2024-12-10 20:14:42,024 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 20:14:42,047 main DEBUG Took 0.020951 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 20:14:42,047 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 20:14:42,053 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 20:14:42,054 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 20:14:42,056 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,065 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 20:14:42,088 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,091 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,093 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,094 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,095 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,098 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,098 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,099 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,100 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,101 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:14:42,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,103 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 20:14:42,105 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:14:42,107 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 20:14:42,109 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 20:14:42,110 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 20:14:42,111 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 20:14:42,112 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 20:14:42,123 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 20:14:42,127 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 20:14:42,129 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 20:14:42,130 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 20:14:42,130 main DEBUG createAppenders(={Console}) 2024-12-10 20:14:42,132 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-10 20:14:42,132 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 20:14:42,133 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-10 20:14:42,133 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 20:14:42,134 main DEBUG OutputStream closed 2024-12-10 20:14:42,134 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 20:14:42,134 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 20:14:42,135 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-10 20:14:42,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 20:14:42,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 20:14:42,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 20:14:42,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 20:14:42,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 20:14:42,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 20:14:42,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 20:14:42,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 20:14:42,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 20:14:42,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 20:14:42,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 20:14:42,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 20:14:42,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 20:14:42,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 20:14:42,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 20:14:42,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 20:14:42,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 20:14:42,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 20:14:42,290 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 20:14:42,293 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-10 20:14:42,293 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 20:14:42,295 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-10T20:14:42,657 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6 2024-12-10 20:14:42,661 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 20:14:42,662 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T20:14:42,676 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-10T20:14:42,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T20:14:42,742 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7, deleteOnExit=true 2024-12-10T20:14:42,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T20:14:42,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/test.cache.data in system properties and HBase conf 2024-12-10T20:14:42,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T20:14:42,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/hadoop.log.dir in system properties and HBase conf 2024-12-10T20:14:42,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T20:14:42,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T20:14:42,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T20:14:42,876 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T20:14:42,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T20:14:42,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T20:14:42,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T20:14:42,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T20:14:42,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T20:14:42,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T20:14:42,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T20:14:42,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T20:14:42,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T20:14:42,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T20:14:42,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/nfs.dump.dir in system properties and HBase conf 2024-12-10T20:14:42,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/java.io.tmpdir in system properties and HBase conf 2024-12-10T20:14:42,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T20:14:42,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T20:14:43,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T20:14:44,350 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T20:14:44,451 INFO [Time-limited test {}] log.Log(170): Logging initialized @3307ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T20:14:44,554 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:14:44,655 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T20:14:44,694 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T20:14:44,695 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T20:14:44,697 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T20:14:44,716 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:14:44,723 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/hadoop.log.dir/,AVAILABLE} 2024-12-10T20:14:44,727 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T20:14:44,998 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/java.io.tmpdir/jetty-localhost-36139-hadoop-hdfs-3_4_1-tests_jar-_-any-8329075297056122091/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T20:14:45,015 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:36139} 2024-12-10T20:14:45,015 INFO [Time-limited test {}] server.Server(415): Started @3872ms 2024-12-10T20:14:45,822 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:14:45,836 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T20:14:45,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T20:14:45,844 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T20:14:45,844 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T20:14:45,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/hadoop.log.dir/,AVAILABLE} 2024-12-10T20:14:45,847 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T20:14:45,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/java.io.tmpdir/jetty-localhost-37193-hadoop-hdfs-3_4_1-tests_jar-_-any-13870911618476103936/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:14:45,986 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:37193} 2024-12-10T20:14:45,986 INFO [Time-limited test {}] server.Server(415): Started @4843ms 2024-12-10T20:14:46,056 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T20:14:47,637 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data1/current/BP-1506670894-172.17.0.3-1733861683702/current, will proceed with Du for space computation calculation, 2024-12-10T20:14:47,637 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2/current/BP-1506670894-172.17.0.3-1733861683702/current, will proceed with Du for space computation calculation, 2024-12-10T20:14:47,710 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T20:14:47,786 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4ba0ce9f968f068 with lease ID 0xb4009df0b007a429: Processing first storage report for DS-68733e57-d57c-4e39-97e9-323c09730199 from datanode DatanodeRegistration(127.0.0.1:46079, datanodeUuid=cd217020-ad99-425b-ab5c-371e4fd24d2e, infoPort=42885, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=1068063302;c=1733861683702) 2024-12-10T20:14:47,787 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4ba0ce9f968f068 with lease ID 0xb4009df0b007a429: from storage DS-68733e57-d57c-4e39-97e9-323c09730199 node DatanodeRegistration(127.0.0.1:46079, datanodeUuid=cd217020-ad99-425b-ab5c-371e4fd24d2e, infoPort=42885, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=1068063302;c=1733861683702), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T20:14:47,788 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4ba0ce9f968f068 with lease ID 0xb4009df0b007a429: Processing first storage report for DS-f84daf2b-9739-4e6e-bfdb-f09369830742 from datanode DatanodeRegistration(127.0.0.1:46079, datanodeUuid=cd217020-ad99-425b-ab5c-371e4fd24d2e, infoPort=42885, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=1068063302;c=1733861683702) 2024-12-10T20:14:47,788 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4ba0ce9f968f068 with lease ID 0xb4009df0b007a429: from storage DS-f84daf2b-9739-4e6e-bfdb-f09369830742 node DatanodeRegistration(127.0.0.1:46079, datanodeUuid=cd217020-ad99-425b-ab5c-371e4fd24d2e, infoPort=42885, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=1068063302;c=1733861683702), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T20:14:47,804 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6 2024-12-10T20:14:47,929 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/zookeeper_0, clientPort=62580, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T20:14:47,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62580 2024-12-10T20:14:47,972 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:47,976 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:48,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741825_1001 (size=7) 2024-12-10T20:14:48,670 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e with version=8 2024-12-10T20:14:48,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/hbase-staging 2024-12-10T20:14:48,761 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T20:14:49,089 INFO [Time-limited test {}] client.ConnectionUtils(128): master/03a2ae042c95:0 server-side Connection retries=45 2024-12-10T20:14:49,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:14:49,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T20:14:49,110 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T20:14:49,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:14:49,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T20:14:49,286 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T20:14:49,347 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T20:14:49,358 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T20:14:49,363 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T20:14:49,395 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5283 (auto-detected) 2024-12-10T20:14:49,396 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-10T20:14:49,423 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45643 2024-12-10T20:14:49,455 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45643 connecting to ZooKeeper ensemble=127.0.0.1:62580 2024-12-10T20:14:49,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456430x0, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T20:14:49,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45643-0x100115dc9260000 connected 2024-12-10T20:14:49,676 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:49,679 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:49,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:14:49,696 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e, hbase.cluster.distributed=false 2024-12-10T20:14:49,724 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T20:14:49,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45643 2024-12-10T20:14:49,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45643 2024-12-10T20:14:49,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45643 2024-12-10T20:14:49,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45643 2024-12-10T20:14:49,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45643 2024-12-10T20:14:49,885 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/03a2ae042c95:0 server-side Connection retries=45 2024-12-10T20:14:49,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:14:49,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T20:14:49,887 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T20:14:49,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:14:49,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T20:14:49,891 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T20:14:49,894 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T20:14:49,900 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38133 2024-12-10T20:14:49,903 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38133 connecting to ZooKeeper ensemble=127.0.0.1:62580 2024-12-10T20:14:49,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:49,908 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:49,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381330x0, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T20:14:49,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381330x0, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:14:49,933 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T20:14:49,936 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38133-0x100115dc9260001 connected 2024-12-10T20:14:49,959 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T20:14:49,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T20:14:49,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T20:14:49,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38133 2024-12-10T20:14:49,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38133 2024-12-10T20:14:49,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38133 2024-12-10T20:14:49,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38133 2024-12-10T20:14:50,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38133 2024-12-10T20:14:50,024 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;03a2ae042c95:45643 2024-12-10T20:14:50,025 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/03a2ae042c95,45643,1733861688870 2024-12-10T20:14:50,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:14:50,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:14:50,046 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/03a2ae042c95,45643,1733861688870 2024-12-10T20:14:50,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:50,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T20:14:50,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:50,084 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T20:14:50,089 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/03a2ae042c95,45643,1733861688870 from backup master directory 2024-12-10T20:14:50,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:14:50,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/03a2ae042c95,45643,1733861688870 2024-12-10T20:14:50,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:14:50,114 WARN [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T20:14:50,117 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=03a2ae042c95,45643,1733861688870 2024-12-10T20:14:50,119 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T20:14:50,129 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T20:14:50,208 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/hbase.id] with ID: 034640c3-bcac-4843-ae54-831f788f9a71 2024-12-10T20:14:50,208 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/.tmp/hbase.id 2024-12-10T20:14:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741826_1002 (size=42) 2024-12-10T20:14:50,234 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/.tmp/hbase.id]:[hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/hbase.id] 2024-12-10T20:14:50,289 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:14:50,295 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T20:14:50,324 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-12-10T20:14:50,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:50,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741827_1003 (size=196) 2024-12-10T20:14:50,387 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:14:50,391 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T20:14:50,409 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:50,413 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:14:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741828_1004 (size=1189) 2024-12-10T20:14:50,467 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store 2024-12-10T20:14:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741829_1005 (size=34) 2024-12-10T20:14:50,895 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T20:14:50,899 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:14:50,900 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T20:14:50,900 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:14:50,901 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:14:50,902 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T20:14:50,903 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:14:50,903 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:14:50,905 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733861690900Disabling compacts and flushes for region at 1733861690900Disabling writes for close at 1733861690903 (+3 ms)Writing region close event to WAL at 1733861690903Closed at 1733861690903 2024-12-10T20:14:50,908 WARN [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/.initializing 2024-12-10T20:14:50,908 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870 2024-12-10T20:14:50,917 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:14:50,933 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C45643%2C1733861688870, suffix=, logDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870, archiveDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/oldWALs, maxLogs=10 2024-12-10T20:14:50,957 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861690938, exclude list is [], retry=0 2024-12-10T20:14:50,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK] 2024-12-10T20:14:50,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T20:14:51,029 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861690938 2024-12-10T20:14:51,030 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42885:42885)] 2024-12-10T20:14:51,031 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:14:51,031 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:14:51,036 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,037 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T20:14:51,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:51,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:51,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T20:14:51,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:51,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:14:51,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T20:14:51,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:51,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:14:51,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T20:14:51,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:51,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:14:51,140 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,143 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,145 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,150 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,151 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,155 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T20:14:51,159 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:14:51,164 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:14:51,166 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71768687, jitterRate=0.06943677365779877}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T20:14:51,170 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733861691052Initializing all the Stores at 1733861691059 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861691059Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861691061 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861691062 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861691062Cleaning up temporary data from old regions at 1733861691151 (+89 ms)Region opened successfully at 1733861691170 (+19 ms) 2024-12-10T20:14:51,172 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T20:14:51,207 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7ae8f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=03a2ae042c95/172.17.0.3:0 2024-12-10T20:14:51,239 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T20:14:51,250 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T20:14:51,250 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T20:14:51,253 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T20:14:51,254 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T20:14:51,259 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-10T20:14:51,259 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T20:14:51,285 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T20:14:51,296 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T20:14:51,408 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T20:14:51,411 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T20:14:51,413 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T20:14:51,421 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T20:14:51,424 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T20:14:51,428 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T20:14:51,440 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T20:14:51,442 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T20:14:51,452 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T20:14:51,474 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T20:14:51,484 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T20:14:51,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T20:14:51,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T20:14:51,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:51,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:51,498 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=03a2ae042c95,45643,1733861688870, sessionid=0x100115dc9260000, setting cluster-up flag (Was=false) 2024-12-10T20:14:51,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:51,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:51,558 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T20:14:51,560 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=03a2ae042c95,45643,1733861688870 2024-12-10T20:14:51,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:51,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:51,610 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T20:14:51,612 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=03a2ae042c95,45643,1733861688870 2024-12-10T20:14:51,619 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T20:14:51,706 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T20:14:51,708 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(746): ClusterId : 034640c3-bcac-4843-ae54-831f788f9a71 2024-12-10T20:14:51,711 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T20:14:51,716 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T20:14:51,723 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T20:14:51,726 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T20:14:51,727 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T20:14:51,729 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 03a2ae042c95,45643,1733861688870 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T20:14:51,738 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T20:14:51,739 DEBUG [RS:0;03a2ae042c95:38133 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308587f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=03a2ae042c95/172.17.0.3:0 2024-12-10T20:14:51,739 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:14:51,740 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:14:51,740 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:14:51,740 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:14:51,740 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/03a2ae042c95:0, corePoolSize=10, maxPoolSize=10 2024-12-10T20:14:51,741 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,741 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/03a2ae042c95:0, corePoolSize=2, maxPoolSize=2 2024-12-10T20:14:51,741 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,749 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T20:14:51,749 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733861721749 2024-12-10T20:14:51,749 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T20:14:51,750 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T20:14:51,752 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T20:14:51,755 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T20:14:51,756 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T20:14:51,756 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T20:14:51,756 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T20:14:51,759 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;03a2ae042c95:38133 2024-12-10T20:14:51,762 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:51,762 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T20:14:51,763 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T20:14:51,763 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T20:14:51,764 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T20:14:51,765 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,767 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(2659): reportForDuty to master=03a2ae042c95,45643,1733861688870 with port=38133, startcode=1733861689839 2024-12-10T20:14:51,773 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T20:14:51,774 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T20:14:51,775 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T20:14:51,787 DEBUG [RS:0;03a2ae042c95:38133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T20:14:51,799 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T20:14:51,799 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T20:14:51,801 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.large.0-1733861691801,5,FailOnTimeoutGroup] 2024-12-10T20:14:51,802 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.small.0-1733861691801,5,FailOnTimeoutGroup] 2024-12-10T20:14:51,803 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,803 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T20:14:51,804 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,805 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741831_1007 (size=1321) 2024-12-10T20:14:51,811 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T20:14:51,812 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:14:51,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46185, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T20:14:51,864 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 03a2ae042c95,38133,1733861689839 2024-12-10T20:14:51,867 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] master.ServerManager(517): Registering regionserver=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:51,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741832_1008 (size=32) 2024-12-10T20:14:51,883 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:14:51,883 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35331 2024-12-10T20:14:51,883 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T20:14:51,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T20:14:51,896 DEBUG [RS:0;03a2ae042c95:38133 {}] zookeeper.ZKUtil(111): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/03a2ae042c95,38133,1733861689839 2024-12-10T20:14:51,897 WARN [RS:0;03a2ae042c95:38133 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T20:14:51,897 INFO [RS:0;03a2ae042c95:38133 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:14:51,897 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839 2024-12-10T20:14:51,898 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [03a2ae042c95,38133,1733861689839] 2024-12-10T20:14:51,928 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T20:14:51,944 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T20:14:51,949 INFO [RS:0;03a2ae042c95:38133 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T20:14:51,950 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,951 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T20:14:51,956 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T20:14:51,958 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,958 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,958 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,959 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,959 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,959 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,961 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0, corePoolSize=2, maxPoolSize=2 2024-12-10T20:14:51,961 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,961 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,961 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,961 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,962 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,962 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:14:51,962 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:14:51,962 DEBUG [RS:0;03a2ae042c95:38133 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:14:51,965 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,965 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,966 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,966 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,966 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,966 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,38133,1733861689839-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T20:14:51,985 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T20:14:51,987 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,38133,1733861689839-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,987 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:51,987 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.Replication(171): 03a2ae042c95,38133,1733861689839 started 2024-12-10T20:14:52,009 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:52,010 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1482): Serving as 03a2ae042c95,38133,1733861689839, RpcServer on 03a2ae042c95/172.17.0.3:38133, sessionid=0x100115dc9260001 2024-12-10T20:14:52,011 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T20:14:52,011 DEBUG [RS:0;03a2ae042c95:38133 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 03a2ae042c95,38133,1733861689839 2024-12-10T20:14:52,011 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,38133,1733861689839' 2024-12-10T20:14:52,011 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T20:14:52,012 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T20:14:52,013 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T20:14:52,013 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T20:14:52,013 DEBUG [RS:0;03a2ae042c95:38133 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 03a2ae042c95,38133,1733861689839 2024-12-10T20:14:52,013 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,38133,1733861689839' 2024-12-10T20:14:52,013 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T20:14:52,014 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T20:14:52,015 DEBUG [RS:0;03a2ae042c95:38133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T20:14:52,015 INFO [RS:0;03a2ae042c95:38133 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T20:14:52,015 INFO [RS:0;03a2ae042c95:38133 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T20:14:52,121 INFO [RS:0;03a2ae042c95:38133 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:14:52,125 INFO [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C38133%2C1733861689839, suffix=, logDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839, archiveDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/oldWALs, maxLogs=32 2024-12-10T20:14:52,141 DEBUG [RS:0;03a2ae042c95:38133 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839/03a2ae042c95%2C38133%2C1733861689839.1733861692127, exclude list is [], retry=0 2024-12-10T20:14:52,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK] 2024-12-10T20:14:52,149 INFO [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839/03a2ae042c95%2C38133%2C1733861689839.1733861692127 2024-12-10T20:14:52,149 DEBUG [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42885:42885)] 2024-12-10T20:14:52,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:14:52,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T20:14:52,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T20:14:52,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T20:14:52,292 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T20:14:52,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T20:14:52,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T20:14:52,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T20:14:52,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T20:14:52,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T20:14:52,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740 2024-12-10T20:14:52,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740 2024-12-10T20:14:52,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T20:14:52,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T20:14:52,308 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:14:52,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T20:14:52,316 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:14:52,317 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71787301, jitterRate=0.06971414387226105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:14:52,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733861692283Initializing all the Stores at 1733861692285 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861692285Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861692285Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861692286 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861692286Cleaning up temporary data from old regions at 1733861692307 (+21 ms)Region opened successfully at 1733861692319 (+12 ms) 2024-12-10T20:14:52,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T20:14:52,320 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T20:14:52,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T20:14:52,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T20:14:52,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T20:14:52,322 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T20:14:52,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733861692320Disabling compacts and flushes for region at 1733861692320Disabling writes for close at 1733861692320Writing region close event to WAL at 1733861692321 (+1 ms)Closed at 1733861692322 (+1 ms) 2024-12-10T20:14:52,326 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T20:14:52,326 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T20:14:52,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T20:14:52,340 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T20:14:52,344 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T20:14:52,497 DEBUG [03a2ae042c95:45643 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T20:14:52,509 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:52,515 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 03a2ae042c95,38133,1733861689839, state=OPENING 2024-12-10T20:14:52,568 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T20:14:52,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:52,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:14:52,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:14:52,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:14:52,581 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T20:14:52,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:14:52,761 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T20:14:52,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T20:14:52,778 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T20:14:52,778 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:14:52,779 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T20:14:52,783 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C38133%2C1733861689839.meta, suffix=.meta, logDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839, archiveDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/oldWALs, maxLogs=32 2024-12-10T20:14:52,800 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839/03a2ae042c95%2C38133%2C1733861689839.meta.1733861692784.meta, exclude list is [], retry=0 2024-12-10T20:14:52,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK] 2024-12-10T20:14:52,807 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/WALs/03a2ae042c95,38133,1733861689839/03a2ae042c95%2C38133%2C1733861689839.meta.1733861692784.meta 2024-12-10T20:14:52,808 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42885:42885)] 2024-12-10T20:14:52,808 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:14:52,810 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T20:14:52,813 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T20:14:52,817 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T20:14:52,821 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T20:14:52,822 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:14:52,822 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T20:14:52,822 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T20:14:52,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T20:14:52,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T20:14:52,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T20:14:52,830 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T20:14:52,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,832 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,832 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T20:14:52,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T20:14:52,834 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T20:14:52,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T20:14:52,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:52,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:14:52,839 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T20:14:52,845 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740 2024-12-10T20:14:52,848 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740 2024-12-10T20:14:52,851 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T20:14:52,851 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T20:14:52,852 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:14:52,855 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T20:14:52,858 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68503641, jitterRate=0.020783796906471252}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:14:52,858 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T20:14:52,860 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733861692823Writing region info on filesystem at 1733861692823Initializing all the Stores at 1733861692825 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861692825Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861692826 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861692826Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733861692826Cleaning up temporary data from old regions at 1733861692851 (+25 ms)Running coprocessor post-open hooks at 1733861692858 (+7 ms)Region opened successfully at 1733861692860 (+2 ms) 2024-12-10T20:14:52,868 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733861692750 2024-12-10T20:14:52,881 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T20:14:52,882 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T20:14:52,885 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:52,887 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 03a2ae042c95,38133,1733861689839, state=OPEN 2024-12-10T20:14:52,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T20:14:52,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T20:14:52,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:14:52,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:14:52,970 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:52,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T20:14:52,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=03a2ae042c95,38133,1733861689839 in 387 msec 2024-12-10T20:14:52,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T20:14:52,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 647 msec 2024-12-10T20:14:52,988 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T20:14:52,989 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T20:14:53,008 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:53,010 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:53,031 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:53,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60107, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:53,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3990 sec 2024-12-10T20:14:53,059 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733861693058, completionTime=-1 2024-12-10T20:14:53,062 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-10T20:14:53,062 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T20:14:53,092 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-10T20:14:53,092 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733861753092 2024-12-10T20:14:53,092 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733861813092 2024-12-10T20:14:53,092 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-10T20:14:53,094 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,45643,1733861688870-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,095 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,45643,1733861688870-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,095 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,45643,1733861688870-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,096 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-03a2ae042c95:45643, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,097 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,098 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,106 DEBUG [master/03a2ae042c95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T20:14:53,126 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.009sec 2024-12-10T20:14:53,128 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T20:14:53,130 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T20:14:53,131 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T20:14:53,131 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T20:14:53,131 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T20:14:53,132 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,45643,1733861688870-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T20:14:53,133 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,45643,1733861688870-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T20:14:53,141 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T20:14:53,142 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T20:14:53,143 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,45643,1733861688870-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:14:53,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d114414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:53,223 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T20:14:53,224 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T20:14:53,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:53,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:53,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2479a655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:53,249 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:53,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23002206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:53,253 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:53,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:53,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:53,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=03a2ae042c95,45643,1733861688870 2024-12-10T20:14:53,300 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=208, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=754, ProcessCount=11, AvailableMemoryMB=716 2024-12-10T20:14:53,308 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:14:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:14:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c1af104 2024-12-10T20:14:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:14:53,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:14:53,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:14:53,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:14:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T20:14:53,378 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:14:53,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-10T20:14:53,381 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:53,383 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:14:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:14:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741835_1011 (size=1061) 2024-12-10T20:14:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:14:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:14:53,802 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:14:53,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741836_1012 (size=53) 2024-12-10T20:14:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:14:54,216 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:14:54,216 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 308bff2a777bdccf66ea752f9872cdd4, disabling compactions & flushes 2024-12-10T20:14:54,216 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,216 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,216 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. after waiting 0 ms 2024-12-10T20:14:54,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,217 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 308bff2a777bdccf66ea752f9872cdd4: Waiting for close lock at 1733861694216Disabling compacts and flushes for region at 1733861694216Disabling writes for close at 1733861694217 (+1 ms)Writing region close event to WAL at 1733861694217Closed at 1733861694217 2024-12-10T20:14:54,220 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:14:54,226 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733861694220"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733861694220"}]},"ts":"1733861694220"} 2024-12-10T20:14:54,232 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:14:54,235 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:14:54,238 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861694235"}]},"ts":"1733861694235"} 2024-12-10T20:14:54,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T20:14:54,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, ASSIGN}] 2024-12-10T20:14:54,253 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, ASSIGN 2024-12-10T20:14:54,256 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, ASSIGN; state=OFFLINE, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=false 2024-12-10T20:14:54,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=308bff2a777bdccf66ea752f9872cdd4, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:54,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, ASSIGN because future has completed 2024-12-10T20:14:54,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:14:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:14:54,579 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,579 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:14:54,580 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,580 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:14:54,580 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,580 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,586 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,589 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:14:54,590 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 308bff2a777bdccf66ea752f9872cdd4 columnFamilyName A 2024-12-10T20:14:54,590 DEBUG [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:54,591 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.HStore(327): Store=308bff2a777bdccf66ea752f9872cdd4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:14:54,592 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,594 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:14:54,594 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 308bff2a777bdccf66ea752f9872cdd4 columnFamilyName B 2024-12-10T20:14:54,595 DEBUG [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:54,595 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.HStore(327): Store=308bff2a777bdccf66ea752f9872cdd4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:14:54,596 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,598 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:14:54,598 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 308bff2a777bdccf66ea752f9872cdd4 columnFamilyName C 2024-12-10T20:14:54,599 DEBUG [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:14:54,600 INFO [StoreOpener-308bff2a777bdccf66ea752f9872cdd4-1 {}] regionserver.HStore(327): Store=308bff2a777bdccf66ea752f9872cdd4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:14:54,600 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,600 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,602 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,603 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,605 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,605 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,606 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:14:54,608 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,612 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:14:54,613 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 308bff2a777bdccf66ea752f9872cdd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63294491, jitterRate=-0.056838586926460266}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:14:54,613 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:54,615 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 308bff2a777bdccf66ea752f9872cdd4: Running coprocessor pre-open hook at 1733861694580Writing region info on filesystem at 1733861694580Initializing all the Stores at 1733861694585 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861694585Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861694586 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861694586Cleaning up temporary data from old regions at 1733861694605 (+19 ms)Running coprocessor post-open hooks at 1733861694613 (+8 ms)Region opened successfully at 1733861694614 (+1 ms) 2024-12-10T20:14:54,617 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., pid=6, masterSystemTime=1733861694570 2024-12-10T20:14:54,620 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,620 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:54,622 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=308bff2a777bdccf66ea752f9872cdd4, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:54,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:14:54,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T20:14:54,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 in 214 msec 2024-12-10T20:14:54,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T20:14:54,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, ASSIGN in 384 msec 2024-12-10T20:14:54,640 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:14:54,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861694640"}]},"ts":"1733861694640"} 2024-12-10T20:14:54,646 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T20:14:54,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:14:54,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2840 sec 2024-12-10T20:14:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:14:55,539 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:14:55,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315339e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75100a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cfb3d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,554 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,562 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:14:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:14:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7bda4d25 2024-12-10T20:14:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:14:55,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:14:55,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f83a6d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,583 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,587 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff18d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,599 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bd06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,607 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af20e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,633 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6af04b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5eb702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32105378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0b2d87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,646 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c270ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c4a9e35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,651 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433011f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,654 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcae178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d091df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4686c825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460c95ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:14:55,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:14:55,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:14:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b04c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:14:55,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:14:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d87de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:14:55,676 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,676 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,676 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,676 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,677 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,678 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,684 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,685 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:14:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,695 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:14:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:14:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:14:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-10T20:14:55,703 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:14:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T20:14:55,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:14:55,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:14:55,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:14:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:14:55,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:14:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:55,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:14:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:14:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:14:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T20:14:55,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T20:14:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:55,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:55,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:55,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/de50ebe607fb4231a2047ecc332aa770 is 50, key is test_row_0/A:col10/1733861695744/Put/seqid=0 2024-12-10T20:14:55,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:55,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741837_1013 (size=12001) 2024-12-10T20:14:55,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/de50ebe607fb4231a2047ecc332aa770 2024-12-10T20:14:56,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861755931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861755923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T20:14:56,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861695781, max timestamp=1733861696000Num uniques 859; 2024-12-10T20:14:56,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:14:56,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861695781, max timestamp=1733861696000Num uniques 859; 2024-12-10T20:14:56,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861695781, max timestamp=1733861696000Num uniques 859; 2024-12-10T20:14:56,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:14:56,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T20:14:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:56,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/11e1764df4af430f9f37e74364fc1f9e is 50, key is test_row_0/B:col10/1733861695744/Put/seqid=0 2024-12-10T20:14:56,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861756145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741838_1014 (size=12001) 2024-12-10T20:14:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861756149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861756150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861756170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861756170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T20:14:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861756269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861756269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861756270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T20:14:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861756378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861756388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T20:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861756489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861756489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861756490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/11e1764df4af430f9f37e74364fc1f9e 2024-12-10T20:14:56,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T20:14:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:56,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/9f5825ffbc284b7284333f0a2dfaadae is 50, key is test_row_0/C:col10/1733861695744/Put/seqid=0 2024-12-10T20:14:56,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741839_1015 (size=12001) 2024-12-10T20:14:56,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/9f5825ffbc284b7284333f0a2dfaadae 2024-12-10T20:14:56,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/de50ebe607fb4231a2047ecc332aa770 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/de50ebe607fb4231a2047ecc332aa770 2024-12-10T20:14:56,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/de50ebe607fb4231a2047ecc332aa770, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T20:14:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/11e1764df4af430f9f37e74364fc1f9e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/11e1764df4af430f9f37e74364fc1f9e 2024-12-10T20:14:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/11e1764df4af430f9f37e74364fc1f9e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T20:14:56,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/9f5825ffbc284b7284333f0a2dfaadae as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9f5825ffbc284b7284333f0a2dfaadae 2024-12-10T20:14:56,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861756688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861756708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9f5825ffbc284b7284333f0a2dfaadae, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T20:14:56,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 308bff2a777bdccf66ea752f9872cdd4 in 972ms, sequenceid=13, compaction requested=false 2024-12-10T20:14:56,733 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T20:14:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:56,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T20:14:56,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:56,741 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-10T20:14:56,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:14:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:14:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:14:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:14:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:14:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:14:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/2778c595e25e466691b67a2001f0383a is 50, key is test_row_0/A:col10/1733861696139/Put/seqid=0 2024-12-10T20:14:56,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741840_1016 (size=12001) 2024-12-10T20:14:56,804 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/2778c595e25e466691b67a2001f0383a 2024-12-10T20:14:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T20:14:56,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/dfe9017e5abe4c5695c9f968af136517 is 50, key is test_row_0/B:col10/1733861696139/Put/seqid=0 2024-12-10T20:14:56,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741841_1017 (size=12001) 2024-12-10T20:14:56,882 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/dfe9017e5abe4c5695c9f968af136517 2024-12-10T20:14:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861756863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861756883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861756891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/83bb9a2f35e74e16ae002da9e4f47677 is 50, key is test_row_0/C:col10/1733861696139/Put/seqid=0 2024-12-10T20:14:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741842_1018 (size=12001) 2024-12-10T20:14:56,943 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/83bb9a2f35e74e16ae002da9e4f47677 2024-12-10T20:14:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/2778c595e25e466691b67a2001f0383a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/2778c595e25e466691b67a2001f0383a 2024-12-10T20:14:56,985 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/2778c595e25e466691b67a2001f0383a, entries=150, sequenceid=48, filesize=11.7 K 2024-12-10T20:14:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/dfe9017e5abe4c5695c9f968af136517 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/dfe9017e5abe4c5695c9f968af136517 2024-12-10T20:14:57,005 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/dfe9017e5abe4c5695c9f968af136517, entries=150, sequenceid=48, filesize=11.7 K 2024-12-10T20:14:57,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/83bb9a2f35e74e16ae002da9e4f47677 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/83bb9a2f35e74e16ae002da9e4f47677 2024-12-10T20:14:57,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861757007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,029 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/83bb9a2f35e74e16ae002da9e4f47677, entries=150, sequenceid=48, filesize=11.7 K 2024-12-10T20:14:57,031 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 308bff2a777bdccf66ea752f9872cdd4 in 291ms, sequenceid=48, compaction requested=false 2024-12-10T20:14:57,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:57,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:57,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-10T20:14:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-10T20:14:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:57,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:14:57,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:14:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:14:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:14:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:57,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T20:14:57,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3270 sec 2024-12-10T20:14:57,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/940b503128504135b48911052a680a3d is 50, key is test_row_0/A:col10/1733861697036/Put/seqid=0 2024-12-10T20:14:57,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3580 sec 2024-12-10T20:14:57,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741843_1019 (size=14341) 2024-12-10T20:14:57,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/940b503128504135b48911052a680a3d 2024-12-10T20:14:57,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/15da92bd72a740a3996337d579df4c0a is 50, key is test_row_0/B:col10/1733861697036/Put/seqid=0 2024-12-10T20:14:57,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741844_1020 (size=12001) 2024-12-10T20:14:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861757135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861757140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861757218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861757218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861757227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861757247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861757258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861757457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861757468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/15da92bd72a740a3996337d579df4c0a 2024-12-10T20:14:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861757547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/77834a5dd28643baa31cd7a4b44784ca is 50, key is test_row_0/C:col10/1733861697036/Put/seqid=0 2024-12-10T20:14:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741845_1021 (size=12001) 2024-12-10T20:14:57,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861757767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861757777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T20:14:57,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:14:57,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:14:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-10T20:14:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T20:14:57,865 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:14:57,867 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:14:57,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:14:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T20:14:57,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/77834a5dd28643baa31cd7a4b44784ca 2024-12-10T20:14:58,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T20:14:58,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:58,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:58,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:14:58,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/940b503128504135b48911052a680a3d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/940b503128504135b48911052a680a3d 2024-12-10T20:14:58,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/940b503128504135b48911052a680a3d, entries=200, sequenceid=62, filesize=14.0 K 2024-12-10T20:14:58,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/15da92bd72a740a3996337d579df4c0a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/15da92bd72a740a3996337d579df4c0a 2024-12-10T20:14:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861758057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/15da92bd72a740a3996337d579df4c0a, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T20:14:58,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/77834a5dd28643baa31cd7a4b44784ca as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/77834a5dd28643baa31cd7a4b44784ca 2024-12-10T20:14:58,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/77834a5dd28643baa31cd7a4b44784ca, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T20:14:58,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 308bff2a777bdccf66ea752f9872cdd4 in 1059ms, sequenceid=62, compaction requested=true 2024-12-10T20:14:58,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:58,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:14:58,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:14:58,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:14:58,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:14:58,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:14:58,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:14:58,115 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:14:58,117 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:14:58,122 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:14:58,125 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:14:58,128 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:14:58,129 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:14:58,129 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:58,129 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:58,129 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/de50ebe607fb4231a2047ecc332aa770, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/2778c595e25e466691b67a2001f0383a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/940b503128504135b48911052a680a3d] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=37.4 K 2024-12-10T20:14:58,130 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/11e1764df4af430f9f37e74364fc1f9e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/dfe9017e5abe4c5695c9f968af136517, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/15da92bd72a740a3996337d579df4c0a] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=35.2 K 2024-12-10T20:14:58,132 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting de50ebe607fb4231a2047ecc332aa770, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733861695729 2024-12-10T20:14:58,133 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e1764df4af430f9f37e74364fc1f9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733861695729 2024-12-10T20:14:58,133 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2778c595e25e466691b67a2001f0383a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733861696129 2024-12-10T20:14:58,136 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe9017e5abe4c5695c9f968af136517, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733861696129 2024-12-10T20:14:58,136 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940b503128504135b48911052a680a3d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861696834 2024-12-10T20:14:58,142 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 15da92bd72a740a3996337d579df4c0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861696834 2024-12-10T20:14:58,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T20:14:58,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:58,186 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T20:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:14:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T20:14:58,190 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#9 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:14:58,191 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/572f9d421ee04e86a376196e191a3437 is 50, key is test_row_0/A:col10/1733861697036/Put/seqid=0 2024-12-10T20:14:58,192 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:14:58,193 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/6692100815ec4bc194b3a4bf20a14ba7 is 50, key is test_row_0/B:col10/1733861697036/Put/seqid=0 2024-12-10T20:14:58,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/dab7ee9f6efd4c33ae2bd6ea2930c650 is 50, key is test_row_0/A:col10/1733861697136/Put/seqid=0 2024-12-10T20:14:58,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741846_1022 (size=12104) 2024-12-10T20:14:58,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741847_1023 (size=12104) 2024-12-10T20:14:58,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:14:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:14:58,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741848_1024 (size=12001) 2024-12-10T20:14:58,251 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/572f9d421ee04e86a376196e191a3437 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/572f9d421ee04e86a376196e191a3437 2024-12-10T20:14:58,279 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into 572f9d421ee04e86a376196e191a3437(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:14:58,280 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:58,282 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=13, startTime=1733861698097; duration=0sec 2024-12-10T20:14:58,283 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:14:58,283 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:14:58,283 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:14:58,287 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:14:58,287 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:14:58,287 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:58,293 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9f5825ffbc284b7284333f0a2dfaadae, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/83bb9a2f35e74e16ae002da9e4f47677, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/77834a5dd28643baa31cd7a4b44784ca] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=35.2 K 2024-12-10T20:14:58,294 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5825ffbc284b7284333f0a2dfaadae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733861695729 2024-12-10T20:14:58,295 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83bb9a2f35e74e16ae002da9e4f47677, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733861696129 2024-12-10T20:14:58,296 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77834a5dd28643baa31cd7a4b44784ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861696834 2024-12-10T20:14:58,319 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#12 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:14:58,319 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/907e9b1e07d949918ac4c69d953df6c5 is 50, key is test_row_0/C:col10/1733861697036/Put/seqid=0 2024-12-10T20:14:58,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741849_1025 (size=12104) 2024-12-10T20:14:58,341 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:14:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861758298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861758346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861758346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861758347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,407 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T20:14:58,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861758458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861758458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861758459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T20:14:58,651 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/dab7ee9f6efd4c33ae2bd6ea2930c650 2024-12-10T20:14:58,659 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/6692100815ec4bc194b3a4bf20a14ba7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/6692100815ec4bc194b3a4bf20a14ba7 2024-12-10T20:14:58,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861758668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861758668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861758668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,675 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 6692100815ec4bc194b3a4bf20a14ba7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:14:58,675 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:58,675 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=13, startTime=1733861698112; duration=0sec 2024-12-10T20:14:58,675 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:14:58,675 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:14:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/35b52394185c43279417fb8d70da2cb7 is 50, key is test_row_0/B:col10/1733861697136/Put/seqid=0 2024-12-10T20:14:58,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741850_1026 (size=12001) 2024-12-10T20:14:58,743 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/907e9b1e07d949918ac4c69d953df6c5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/907e9b1e07d949918ac4c69d953df6c5 2024-12-10T20:14:58,759 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into 907e9b1e07d949918ac4c69d953df6c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:14:58,759 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:58,759 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=13, startTime=1733861698112; duration=0sec 2024-12-10T20:14:58,760 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:14:58,760 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:14:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861758977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861758977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:58,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861758978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T20:14:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861759077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:59,084 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:14:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:59,093 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/35b52394185c43279417fb8d70da2cb7 2024-12-10T20:14:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/6f8276f6e02042fcb61b1b9e11f2f517 is 50, key is test_row_0/C:col10/1733861697136/Put/seqid=0 2024-12-10T20:14:59,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741851_1027 (size=12001) 2024-12-10T20:14:59,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T20:14:59,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T20:14:59,345 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T20:14:59,345 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T20:14:59,346 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T20:14:59,346 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T20:14:59,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T20:14:59,347 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T20:14:59,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861759369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:59,376 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861759487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861759487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:14:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861759497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:14:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:14:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:14:59,520 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/6f8276f6e02042fcb61b1b9e11f2f517 2024-12-10T20:14:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/dab7ee9f6efd4c33ae2bd6ea2930c650 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/dab7ee9f6efd4c33ae2bd6ea2930c650 2024-12-10T20:14:59,545 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/dab7ee9f6efd4c33ae2bd6ea2930c650, entries=150, sequenceid=80, filesize=11.7 K 2024-12-10T20:14:59,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/35b52394185c43279417fb8d70da2cb7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/35b52394185c43279417fb8d70da2cb7 2024-12-10T20:14:59,566 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/35b52394185c43279417fb8d70da2cb7, entries=150, sequenceid=80, filesize=11.7 K 2024-12-10T20:14:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/6f8276f6e02042fcb61b1b9e11f2f517 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6f8276f6e02042fcb61b1b9e11f2f517 2024-12-10T20:14:59,580 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6f8276f6e02042fcb61b1b9e11f2f517, entries=150, sequenceid=80, filesize=11.7 K 2024-12-10T20:14:59,583 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 308bff2a777bdccf66ea752f9872cdd4 in 1397ms, sequenceid=80, compaction requested=false 2024-12-10T20:14:59,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:14:59,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:14:59,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-10T20:14:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-10T20:14:59,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-10T20:14:59,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-12-10T20:14:59,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7380 sec 2024-12-10T20:15:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T20:15:00,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:00,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-10T20:15:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T20:15:00,022 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:00,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:00,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T20:15:00,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T20:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,178 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T20:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/02508363e56f40faa7b676173408e1ef is 50, key is test_row_0/A:col10/1733861698314/Put/seqid=0 2024-12-10T20:15:00,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741852_1028 (size=12001) 2024-12-10T20:15:00,218 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/02508363e56f40faa7b676173408e1ef 2024-12-10T20:15:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e329ccce0ad441299635b92b952f4e8b is 50, key is test_row_0/B:col10/1733861698314/Put/seqid=0 2024-12-10T20:15:00,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741853_1029 (size=12001) 2024-12-10T20:15:00,271 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e329ccce0ad441299635b92b952f4e8b 2024-12-10T20:15:00,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/4dbcb9adaab147b493a0a8b5aa445a64 is 50, key is test_row_0/C:col10/1733861698314/Put/seqid=0 2024-12-10T20:15:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741854_1030 (size=12001) 2024-12-10T20:15:00,321 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/4dbcb9adaab147b493a0a8b5aa445a64 2024-12-10T20:15:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/02508363e56f40faa7b676173408e1ef as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/02508363e56f40faa7b676173408e1ef 2024-12-10T20:15:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T20:15:00,347 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/02508363e56f40faa7b676173408e1ef, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T20:15:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e329ccce0ad441299635b92b952f4e8b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e329ccce0ad441299635b92b952f4e8b 2024-12-10T20:15:00,378 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e329ccce0ad441299635b92b952f4e8b, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T20:15:00,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/4dbcb9adaab147b493a0a8b5aa445a64 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/4dbcb9adaab147b493a0a8b5aa445a64 2024-12-10T20:15:00,391 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/4dbcb9adaab147b493a0a8b5aa445a64, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T20:15:00,393 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 308bff2a777bdccf66ea752f9872cdd4 in 215ms, sequenceid=97, compaction requested=true 2024-12-10T20:15:00,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-10T20:15:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-10T20:15:00,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-10T20:15:00,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 373 msec 2024-12-10T20:15:00,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 384 msec 2024-12-10T20:15:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:00,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:15:00,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:00,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:00,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:00,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:00,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:00,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:00,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/817f676b169d408586cf76585602d261 is 50, key is test_row_0/A:col10/1733861700511/Put/seqid=0 2024-12-10T20:15:00,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741855_1031 (size=12001) 2024-12-10T20:15:00,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/817f676b169d408586cf76585602d261 2024-12-10T20:15:00,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c43f0c447d0242cfa859e5a7dc072a99 is 50, key is test_row_0/B:col10/1733861700511/Put/seqid=0 2024-12-10T20:15:00,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861700564, max timestamp=1733861700579Num uniques 859; 2024-12-10T20:15:00,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:00,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861700564, max timestamp=1733861700579Num uniques 859; 2024-12-10T20:15:00,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861700564, max timestamp=1733861700587Num uniques 859; 2024-12-10T20:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:00,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741856_1032 (size=12001) 2024-12-10T20:15:00,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c43f0c447d0242cfa859e5a7dc072a99 2024-12-10T20:15:00,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/33843b73e837426eadd765c7a098fe02 is 50, key is test_row_0/C:col10/1733861700511/Put/seqid=0 2024-12-10T20:15:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T20:15:00,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:00,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-10T20:15:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861760640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T20:15:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861760645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,651 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,652 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:00,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:00,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861760650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741857_1033 (size=12001) 2024-12-10T20:15:00,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/33843b73e837426eadd765c7a098fe02 2024-12-10T20:15:00,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/817f676b169d408586cf76585602d261 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/817f676b169d408586cf76585602d261 2024-12-10T20:15:00,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/817f676b169d408586cf76585602d261, entries=150, sequenceid=107, filesize=11.7 K 2024-12-10T20:15:00,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c43f0c447d0242cfa859e5a7dc072a99 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c43f0c447d0242cfa859e5a7dc072a99 2024-12-10T20:15:00,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c43f0c447d0242cfa859e5a7dc072a99, entries=150, sequenceid=107, filesize=11.7 K 2024-12-10T20:15:00,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/33843b73e837426eadd765c7a098fe02 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/33843b73e837426eadd765c7a098fe02 2024-12-10T20:15:00,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/33843b73e837426eadd765c7a098fe02, entries=150, sequenceid=107, filesize=11.7 K 2024-12-10T20:15:00,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 308bff2a777bdccf66ea752f9872cdd4 in 213ms, sequenceid=107, compaction requested=true 2024-12-10T20:15:00,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:00,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:00,736 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:00,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:00,736 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:00,740 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:00,740 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:00,740 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:00,740 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:00,740 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,740 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,740 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/6692100815ec4bc194b3a4bf20a14ba7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/35b52394185c43279417fb8d70da2cb7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e329ccce0ad441299635b92b952f4e8b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c43f0c447d0242cfa859e5a7dc072a99] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=47.0 K 2024-12-10T20:15:00,740 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/572f9d421ee04e86a376196e191a3437, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/dab7ee9f6efd4c33ae2bd6ea2930c650, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/02508363e56f40faa7b676173408e1ef, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/817f676b169d408586cf76585602d261] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=47.0 K 2024-12-10T20:15:00,744 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572f9d421ee04e86a376196e191a3437, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861696834 2024-12-10T20:15:00,744 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6692100815ec4bc194b3a4bf20a14ba7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861696834 2024-12-10T20:15:00,745 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b52394185c43279417fb8d70da2cb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733861697102 2024-12-10T20:15:00,745 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting dab7ee9f6efd4c33ae2bd6ea2930c650, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733861697102 2024-12-10T20:15:00,746 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02508363e56f40faa7b676173408e1ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733861698288 2024-12-10T20:15:00,747 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e329ccce0ad441299635b92b952f4e8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733861698288 2024-12-10T20:15:00,747 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 817f676b169d408586cf76585602d261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733861700511 2024-12-10T20:15:00,748 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c43f0c447d0242cfa859e5a7dc072a99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733861700511 2024-12-10T20:15:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T20:15:00,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861700630, max timestamp=1733861700762Num uniques 1018; 2024-12-10T20:15:00,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T20:15:00,772 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:00,772 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/ab9a5cb4a2b641b6b34c1f8b53fbd163 is 50, key is test_row_0/B:col10/1733861700511/Put/seqid=0 2024-12-10T20:15:00,776 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:00,777 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f470cb5477ec48eb835a86060ea8dbea is 50, key is test_row_0/A:col10/1733861700511/Put/seqid=0 2024-12-10T20:15:00,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=109.58 KB heapSize=312.13 KB 2024-12-10T20:15:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T20:15:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T20:15:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:00,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741858_1034 (size=12241) 2024-12-10T20:15:00,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/06fac488dd7e4da69f04c48884b3a201 is 50, key is test_row_0/A:col10/1733861700767/Put/seqid=0 2024-12-10T20:15:00,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741859_1035 (size=12241) 2024-12-10T20:15:00,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T20:15:00,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:00,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:00,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:00,820 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f470cb5477ec48eb835a86060ea8dbea as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f470cb5477ec48eb835a86060ea8dbea 2024-12-10T20:15:00,831 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into f470cb5477ec48eb835a86060ea8dbea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:00,831 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:00,831 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=12, startTime=1733861700736; duration=0sec 2024-12-10T20:15:00,831 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:00,832 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:00,832 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:00,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741860_1036 (size=12151) 2024-12-10T20:15:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861760821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,834 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,834 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861760823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,834 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,835 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/907e9b1e07d949918ac4c69d953df6c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6f8276f6e02042fcb61b1b9e11f2f517, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/4dbcb9adaab147b493a0a8b5aa445a64, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/33843b73e837426eadd765c7a098fe02] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=47.0 K 2024-12-10T20:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/06fac488dd7e4da69f04c48884b3a201 2024-12-10T20:15:00,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861760824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,838 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907e9b1e07d949918ac4c69d953df6c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861696834 2024-12-10T20:15:00,838 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f8276f6e02042fcb61b1b9e11f2f517, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733861697102 2024-12-10T20:15:00,840 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dbcb9adaab147b493a0a8b5aa445a64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733861698288 2024-12-10T20:15:00,842 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33843b73e837426eadd765c7a098fe02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733861700511 2024-12-10T20:15:00,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/4bf394ee7f42469d841b130e8af4be7a is 50, key is test_row_0/B:col10/1733861700767/Put/seqid=0 2024-12-10T20:15:00,879 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:00,880 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/bfd600d14f674a1b82699012c75781e1 is 50, key is test_row_0/C:col10/1733861700511/Put/seqid=0 2024-12-10T20:15:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741861_1037 (size=12151) 2024-12-10T20:15:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861760937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741862_1038 (size=12241) 2024-12-10T20:15:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861760949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861760947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:00,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T20:15:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,966 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/bfd600d14f674a1b82699012c75781e1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/bfd600d14f674a1b82699012c75781e1 2024-12-10T20:15:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:00,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T20:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:00,982 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into bfd600d14f674a1b82699012c75781e1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:00,982 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:00,982 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=12, startTime=1733861700738; duration=0sec 2024-12-10T20:15:00,982 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:00,982 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861761107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,110 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T20:15:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:01,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:01,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861761147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861761157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861761157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,214 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/ab9a5cb4a2b641b6b34c1f8b53fbd163 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/ab9a5cb4a2b641b6b34c1f8b53fbd163 2024-12-10T20:15:01,229 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into ab9a5cb4a2b641b6b34c1f8b53fbd163(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:01,229 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:01,229 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=12, startTime=1733861700736; duration=0sec 2024-12-10T20:15:01,230 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:01,230 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:01,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T20:15:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:01,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T20:15:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:01,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/4bf394ee7f42469d841b130e8af4be7a 2024-12-10T20:15:01,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/c8517c4f92ce4a4faec2d3aae420bafd is 50, key is test_row_0/C:col10/1733861700767/Put/seqid=0 2024-12-10T20:15:01,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741863_1039 (size=12151) 2024-12-10T20:15:01,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/c8517c4f92ce4a4faec2d3aae420bafd 2024-12-10T20:15:01,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/06fac488dd7e4da69f04c48884b3a201 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/06fac488dd7e4da69f04c48884b3a201 2024-12-10T20:15:01,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/06fac488dd7e4da69f04c48884b3a201, entries=150, sequenceid=146, filesize=11.9 K 2024-12-10T20:15:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/4bf394ee7f42469d841b130e8af4be7a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4bf394ee7f42469d841b130e8af4be7a 2024-12-10T20:15:01,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861761387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,389 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:01,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4bf394ee7f42469d841b130e8af4be7a, entries=150, sequenceid=146, filesize=11.9 K 2024-12-10T20:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/c8517c4f92ce4a4faec2d3aae420bafd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c8517c4f92ce4a4faec2d3aae420bafd 2024-12-10T20:15:01,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c8517c4f92ce4a4faec2d3aae420bafd, entries=150, sequenceid=146, filesize=11.9 K 2024-12-10T20:15:01,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~312.08 KB/319568, currentSize=67.09 KB/68700 for 308bff2a777bdccf66ea752f9872cdd4 in 627ms, sequenceid=146, compaction requested=false 2024-12-10T20:15:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:01,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T20:15:01,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:01,432 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:15:01,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:01,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:01,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ce66aff9f2d6447f9ee8100cff8ce1f0 is 50, key is test_row_0/A:col10/1733861700794/Put/seqid=0 2024-12-10T20:15:01,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741864_1040 (size=12151) 2024-12-10T20:15:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861761536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861761541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861761537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861761647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861761647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861761657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T20:15:01,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861761857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861761857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861761867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:01,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:01,884 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ce66aff9f2d6447f9ee8100cff8ce1f0 2024-12-10T20:15:01,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/097fdcac26e4409d96327efe0a1faa08 is 50, key is test_row_0/B:col10/1733861700794/Put/seqid=0 2024-12-10T20:15:01,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741865_1041 (size=12151) 2024-12-10T20:15:01,921 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/097fdcac26e4409d96327efe0a1faa08 2024-12-10T20:15:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/a3f4c9baae614158ad112d6e525b82c6 is 50, key is test_row_0/C:col10/1733861700794/Put/seqid=0 2024-12-10T20:15:01,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741866_1042 (size=12151) 2024-12-10T20:15:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861762167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861762167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861762177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,380 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/a3f4c9baae614158ad112d6e525b82c6 2024-12-10T20:15:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ce66aff9f2d6447f9ee8100cff8ce1f0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ce66aff9f2d6447f9ee8100cff8ce1f0 2024-12-10T20:15:02,394 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ce66aff9f2d6447f9ee8100cff8ce1f0, entries=150, sequenceid=162, filesize=11.9 K 2024-12-10T20:15:02,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/097fdcac26e4409d96327efe0a1faa08 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/097fdcac26e4409d96327efe0a1faa08 2024-12-10T20:15:02,403 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/097fdcac26e4409d96327efe0a1faa08, entries=150, sequenceid=162, filesize=11.9 K 2024-12-10T20:15:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/a3f4c9baae614158ad112d6e525b82c6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/a3f4c9baae614158ad112d6e525b82c6 2024-12-10T20:15:02,416 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/a3f4c9baae614158ad112d6e525b82c6, entries=150, sequenceid=162, filesize=11.9 K 2024-12-10T20:15:02,418 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 308bff2a777bdccf66ea752f9872cdd4 in 985ms, sequenceid=162, compaction requested=true 2024-12-10T20:15:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-10T20:15:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-10T20:15:02,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T20:15:02,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-12-10T20:15:02,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7790 sec 2024-12-10T20:15:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:02,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T20:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:02,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ac00d222de24487e8e66d2d87e8c6f2b is 50, key is test_row_0/A:col10/1733861702679/Put/seqid=0 2024-12-10T20:15:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861762703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741867_1043 (size=14541) 2024-12-10T20:15:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861762707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861762707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T20:15:02,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:02,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-10T20:15:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T20:15:02,800 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:02,801 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:02,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861762817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861762817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861762827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T20:15:02,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T20:15:02,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:02,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861763027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861763027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861763036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T20:15:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T20:15:03,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ac00d222de24487e8e66d2d87e8c6f2b 2024-12-10T20:15:03,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/477626c127614b9c960edf51781c9e26 is 50, key is test_row_0/B:col10/1733861702679/Put/seqid=0 2024-12-10T20:15:03,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741868_1044 (size=12151) 2024-12-10T20:15:03,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/477626c127614b9c960edf51781c9e26 2024-12-10T20:15:03,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d6ddcd9fa308431691b18ddc47c565cf is 50, key is test_row_0/C:col10/1733861702679/Put/seqid=0 2024-12-10T20:15:03,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741869_1045 (size=12151) 2024-12-10T20:15:03,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T20:15:03,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:03,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861763337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861763337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861763347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T20:15:03,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T20:15:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T20:15:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:03,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d6ddcd9fa308431691b18ddc47c565cf 2024-12-10T20:15:03,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ac00d222de24487e8e66d2d87e8c6f2b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ac00d222de24487e8e66d2d87e8c6f2b 2024-12-10T20:15:03,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ac00d222de24487e8e66d2d87e8c6f2b, entries=200, sequenceid=182, filesize=14.2 K 2024-12-10T20:15:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/477626c127614b9c960edf51781c9e26 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/477626c127614b9c960edf51781c9e26 2024-12-10T20:15:03,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/477626c127614b9c960edf51781c9e26, entries=150, sequenceid=182, filesize=11.9 K 2024-12-10T20:15:03,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d6ddcd9fa308431691b18ddc47c565cf as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d6ddcd9fa308431691b18ddc47c565cf 2024-12-10T20:15:03,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d6ddcd9fa308431691b18ddc47c565cf, entries=150, sequenceid=182, filesize=11.9 K 2024-12-10T20:15:03,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 308bff2a777bdccf66ea752f9872cdd4 in 1013ms, sequenceid=182, compaction requested=true 2024-12-10T20:15:03,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:03,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:03,694 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:03,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:03,694 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:03,696 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:03,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:03,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:03,696 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:03,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:03,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:03,696 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,696 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/ab9a5cb4a2b641b6b34c1f8b53fbd163, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4bf394ee7f42469d841b130e8af4be7a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/097fdcac26e4409d96327efe0a1faa08, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/477626c127614b9c960edf51781c9e26] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=47.6 K 2024-12-10T20:15:03,697 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:03,697 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9a5cb4a2b641b6b34c1f8b53fbd163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733861700511 2024-12-10T20:15:03,697 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:03,697 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,697 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f470cb5477ec48eb835a86060ea8dbea, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/06fac488dd7e4da69f04c48884b3a201, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ce66aff9f2d6447f9ee8100cff8ce1f0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ac00d222de24487e8e66d2d87e8c6f2b] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=49.9 K 2024-12-10T20:15:03,697 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf394ee7f42469d841b130e8af4be7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733861700630 2024-12-10T20:15:03,698 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f470cb5477ec48eb835a86060ea8dbea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733861700511 2024-12-10T20:15:03,698 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 097fdcac26e4409d96327efe0a1faa08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733861700794 2024-12-10T20:15:03,698 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06fac488dd7e4da69f04c48884b3a201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733861700630 2024-12-10T20:15:03,699 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce66aff9f2d6447f9ee8100cff8ce1f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733861700794 2024-12-10T20:15:03,699 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac00d222de24487e8e66d2d87e8c6f2b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733861701530 2024-12-10T20:15:03,701 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 477626c127614b9c960edf51781c9e26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733861701530 2024-12-10T20:15:03,717 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:03,717 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/81b55e9f84934e4897d93136daed98af is 50, key is test_row_0/A:col10/1733861702679/Put/seqid=0 2024-12-10T20:15:03,721 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:03,722 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/512b64b25d53482c8926284a4bfee3b4 is 50, key is test_row_0/B:col10/1733861702679/Put/seqid=0 2024-12-10T20:15:03,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741871_1047 (size=12527) 2024-12-10T20:15:03,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741870_1046 (size=12527) 2024-12-10T20:15:03,743 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/512b64b25d53482c8926284a4bfee3b4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/512b64b25d53482c8926284a4bfee3b4 2024-12-10T20:15:03,754 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/81b55e9f84934e4897d93136daed98af as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/81b55e9f84934e4897d93136daed98af 2024-12-10T20:15:03,760 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 512b64b25d53482c8926284a4bfee3b4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:03,760 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:03,760 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=12, startTime=1733861703694; duration=0sec 2024-12-10T20:15:03,760 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:03,761 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:03,761 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:03,764 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:03,764 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:03,764 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,764 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/bfd600d14f674a1b82699012c75781e1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c8517c4f92ce4a4faec2d3aae420bafd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/a3f4c9baae614158ad112d6e525b82c6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d6ddcd9fa308431691b18ddc47c565cf] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=47.6 K 2024-12-10T20:15:03,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T20:15:03,766 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into 81b55e9f84934e4897d93136daed98af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:03,766 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:03,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:03,766 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=12, startTime=1733861703693; duration=0sec 2024-12-10T20:15:03,766 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T20:15:03,766 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:03,766 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:03,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:03,767 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting bfd600d14f674a1b82699012c75781e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733861700511 2024-12-10T20:15:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:03,767 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c8517c4f92ce4a4faec2d3aae420bafd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733861700630 2024-12-10T20:15:03,768 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f4c9baae614158ad112d6e525b82c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733861700794 2024-12-10T20:15:03,769 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ddcd9fa308431691b18ddc47c565cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733861701530 2024-12-10T20:15:03,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/71006d0c0a8e4aa08a29653b9a13d100 is 50, key is test_row_0/A:col10/1733861702696/Put/seqid=0 2024-12-10T20:15:03,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741872_1048 (size=12151) 2024-12-10T20:15:03,788 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/71006d0c0a8e4aa08a29653b9a13d100 2024-12-10T20:15:03,805 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:03,806 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/171f8f9b64d841cebad3f6e7ef1a421c is 50, key is test_row_0/C:col10/1733861702679/Put/seqid=0 2024-12-10T20:15:03,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/40b6246d39af423894dd7ffa86e5b807 is 50, key is test_row_0/B:col10/1733861702696/Put/seqid=0 2024-12-10T20:15:03,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741873_1049 (size=12527) 2024-12-10T20:15:03,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741874_1050 (size=12151) 2024-12-10T20:15:03,829 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/40b6246d39af423894dd7ffa86e5b807 2024-12-10T20:15:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/858d9e974b8448599642c71fc40336b3 is 50, key is test_row_0/C:col10/1733861702696/Put/seqid=0 2024-12-10T20:15:03,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741875_1051 (size=12151) 2024-12-10T20:15:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:03,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:03,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861763889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861763890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861763894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T20:15:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861764007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861764007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861764007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861764217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861764225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861764225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,239 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/171f8f9b64d841cebad3f6e7ef1a421c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/171f8f9b64d841cebad3f6e7ef1a421c 2024-12-10T20:15:04,251 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into 171f8f9b64d841cebad3f6e7ef1a421c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:04,251 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:04,251 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=12, startTime=1733861703696; duration=0sec 2024-12-10T20:15:04,251 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/858d9e974b8448599642c71fc40336b3 2024-12-10T20:15:04,251 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:04,251 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/71006d0c0a8e4aa08a29653b9a13d100 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/71006d0c0a8e4aa08a29653b9a13d100 2024-12-10T20:15:04,269 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/71006d0c0a8e4aa08a29653b9a13d100, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T20:15:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/40b6246d39af423894dd7ffa86e5b807 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/40b6246d39af423894dd7ffa86e5b807 2024-12-10T20:15:04,285 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/40b6246d39af423894dd7ffa86e5b807, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T20:15:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/858d9e974b8448599642c71fc40336b3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/858d9e974b8448599642c71fc40336b3 2024-12-10T20:15:04,299 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/858d9e974b8448599642c71fc40336b3, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T20:15:04,300 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 308bff2a777bdccf66ea752f9872cdd4 in 534ms, sequenceid=196, compaction requested=false 2024-12-10T20:15:04,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:04,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:04,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-10T20:15:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-10T20:15:04,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-10T20:15:04,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-12-10T20:15:04,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5110 sec 2024-12-10T20:15:04,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861703876, max timestamp=1733861704529Num uniques 859; 2024-12-10T20:15:04,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:04,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-10T20:15:04,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:04,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:04,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:04,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:04,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:04,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:04,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c8510dd1824a4a749bd4132b536098ad is 50, key is test_row_0/A:col10/1733861703887/Put/seqid=0 2024-12-10T20:15:04,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741876_1052 (size=14541) 2024-12-10T20:15:04,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c8510dd1824a4a749bd4132b536098ad 2024-12-10T20:15:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861764597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861764600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/1e349201a3f040cbb6c590f121f4dd14 is 50, key is test_row_0/B:col10/1733861703887/Put/seqid=0 2024-12-10T20:15:04,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861764607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741877_1053 (size=12151) 2024-12-10T20:15:04,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/1e349201a3f040cbb6c590f121f4dd14 2024-12-10T20:15:04,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/c39fe1aacfaf4ac69bf1f96123270dc5 is 50, key is test_row_0/C:col10/1733861703887/Put/seqid=0 2024-12-10T20:15:04,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741878_1054 (size=12151) 2024-12-10T20:15:04,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/c39fe1aacfaf4ac69bf1f96123270dc5 2024-12-10T20:15:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861764707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861764718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861764728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:04,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c8510dd1824a4a749bd4132b536098ad as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c8510dd1824a4a749bd4132b536098ad 2024-12-10T20:15:04,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c8510dd1824a4a749bd4132b536098ad, entries=200, sequenceid=218, filesize=14.2 K 2024-12-10T20:15:04,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/1e349201a3f040cbb6c590f121f4dd14 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/1e349201a3f040cbb6c590f121f4dd14 2024-12-10T20:15:04,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/1e349201a3f040cbb6c590f121f4dd14, entries=150, sequenceid=218, filesize=11.9 K 2024-12-10T20:15:04,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/c39fe1aacfaf4ac69bf1f96123270dc5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c39fe1aacfaf4ac69bf1f96123270dc5 2024-12-10T20:15:04,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c39fe1aacfaf4ac69bf1f96123270dc5, entries=150, sequenceid=218, filesize=11.9 K 2024-12-10T20:15:04,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=93.93 KB/96180 for 308bff2a777bdccf66ea752f9872cdd4 in 270ms, sequenceid=218, compaction requested=true 2024-12-10T20:15:04,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:04,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:04,804 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:04,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:04,804 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:04,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:04,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:04,805 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:04,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:04,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:04,805 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:04,805 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:04,806 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/81b55e9f84934e4897d93136daed98af, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/71006d0c0a8e4aa08a29653b9a13d100, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c8510dd1824a4a749bd4132b536098ad] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=38.3 K 2024-12-10T20:15:04,806 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:04,806 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:04,806 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:04,806 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/512b64b25d53482c8926284a4bfee3b4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/40b6246d39af423894dd7ffa86e5b807, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/1e349201a3f040cbb6c590f121f4dd14] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=36.0 K 2024-12-10T20:15:04,807 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b55e9f84934e4897d93136daed98af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733861701530 2024-12-10T20:15:04,807 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 512b64b25d53482c8926284a4bfee3b4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733861701530 2024-12-10T20:15:04,807 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71006d0c0a8e4aa08a29653b9a13d100, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733861702696 2024-12-10T20:15:04,808 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b6246d39af423894dd7ffa86e5b807, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733861702696 2024-12-10T20:15:04,808 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8510dd1824a4a749bd4132b536098ad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733861703876 2024-12-10T20:15:04,808 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e349201a3f040cbb6c590f121f4dd14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733861703876 2024-12-10T20:15:04,835 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:04,836 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c5291c077b4f4eada909bbd22b20922a is 50, key is test_row_0/A:col10/1733861703887/Put/seqid=0 2024-12-10T20:15:04,863 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:04,863 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e10512b8e7a54972b99497625b691058 is 50, key is test_row_0/B:col10/1733861703887/Put/seqid=0 2024-12-10T20:15:04,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741879_1055 (size=12629) 2024-12-10T20:15:04,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741880_1056 (size=12629) 2024-12-10T20:15:04,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T20:15:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:04,944 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e10512b8e7a54972b99497625b691058 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e10512b8e7a54972b99497625b691058 2024-12-10T20:15:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T20:15:04,949 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:04,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:04,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/003aa63ed6304117b80d5481a087822d is 50, key is test_row_0/A:col10/1733861704928/Put/seqid=0 2024-12-10T20:15:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-10T20:15:04,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T20:15:04,954 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:04,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741881_1057 (size=14541) 2024-12-10T20:15:04,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/003aa63ed6304117b80d5481a087822d 2024-12-10T20:15:04,990 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into e10512b8e7a54972b99497625b691058(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:04,990 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:04,991 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=13, startTime=1733861704804; duration=0sec 2024-12-10T20:15:04,991 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:04,991 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:04,991 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:04,993 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:04,993 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:04,993 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:04,993 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/171f8f9b64d841cebad3f6e7ef1a421c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/858d9e974b8448599642c71fc40336b3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c39fe1aacfaf4ac69bf1f96123270dc5] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=36.0 K 2024-12-10T20:15:04,996 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 171f8f9b64d841cebad3f6e7ef1a421c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733861701530 2024-12-10T20:15:04,996 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 858d9e974b8448599642c71fc40336b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733861702696 2024-12-10T20:15:04,997 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c39fe1aacfaf4ac69bf1f96123270dc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733861703876 2024-12-10T20:15:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861764995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861765007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861765010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/d1edc5193e2845aeacc5f6980218cfde is 50, key is test_row_0/B:col10/1733861704928/Put/seqid=0 2024-12-10T20:15:05,027 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:05,028 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5 is 50, key is test_row_0/C:col10/1733861703887/Put/seqid=0 2024-12-10T20:15:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T20:15:05,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741882_1058 (size=12151) 2024-12-10T20:15:05,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741883_1059 (size=12629) 2024-12-10T20:15:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,123 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5 2024-12-10T20:15:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861765117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861765118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861765118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,138 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into cfb8541bc4e84a2bb1a0d2cbf590c4e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:05,138 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:05,138 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=13, startTime=1733861704805; duration=0sec 2024-12-10T20:15:05,140 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:05,140 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:05,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861765138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,147 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T20:15:05,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,318 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c5291c077b4f4eada909bbd22b20922a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5291c077b4f4eada909bbd22b20922a 2024-12-10T20:15:05,325 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into c5291c077b4f4eada909bbd22b20922a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:05,325 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:05,325 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=13, startTime=1733861704804; duration=0sec 2024-12-10T20:15:05,325 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:05,325 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:05,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861765327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861765336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861765336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861765417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,419 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/d1edc5193e2845aeacc5f6980218cfde 2024-12-10T20:15:05,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/208fca06b9ed49ea80e4c83ff32f5cde is 50, key is test_row_0/C:col10/1733861704928/Put/seqid=0 2024-12-10T20:15:05,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741884_1060 (size=12151) 2024-12-10T20:15:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T20:15:05,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861765637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861765647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861765647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:05,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:05,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/208fca06b9ed49ea80e4c83ff32f5cde 2024-12-10T20:15:05,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:05,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:05,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:05,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/003aa63ed6304117b80d5481a087822d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/003aa63ed6304117b80d5481a087822d 2024-12-10T20:15:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:05,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/003aa63ed6304117b80d5481a087822d, entries=200, sequenceid=236, filesize=14.2 K 2024-12-10T20:15:05,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/d1edc5193e2845aeacc5f6980218cfde as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/d1edc5193e2845aeacc5f6980218cfde 2024-12-10T20:15:05,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/d1edc5193e2845aeacc5f6980218cfde, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T20:15:05,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/208fca06b9ed49ea80e4c83ff32f5cde as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/208fca06b9ed49ea80e4c83ff32f5cde 2024-12-10T20:15:05,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/208fca06b9ed49ea80e4c83ff32f5cde, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T20:15:05,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 308bff2a777bdccf66ea752f9872cdd4 in 999ms, sequenceid=236, compaction requested=false 2024-12-10T20:15:05,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:06,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T20:15:06,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:06,060 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T20:15:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/9bd10d3dc2c94e2a9b178316945d4b5a is 50, key is test_row_0/A:col10/1733861705007/Put/seqid=0 2024-12-10T20:15:06,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741885_1061 (size=12151) 2024-12-10T20:15:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T20:15:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:06,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:06,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861766183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861766188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861766188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861766297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861766297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861766298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,473 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/9bd10d3dc2c94e2a9b178316945d4b5a 2024-12-10T20:15:06,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/5b658b1c442d45cab8932b78c3843632 is 50, key is test_row_0/B:col10/1733861705007/Put/seqid=0 2024-12-10T20:15:06,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861766508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861766508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861766508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741886_1062 (size=12151) 2024-12-10T20:15:06,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861766818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861766818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861766821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:06,931 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/5b658b1c442d45cab8932b78c3843632 2024-12-10T20:15:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/3271212c956648b89572900fc521f3be is 50, key is test_row_0/C:col10/1733861705007/Put/seqid=0 2024-12-10T20:15:06,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741887_1063 (size=12151) 2024-12-10T20:15:07,002 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/3271212c956648b89572900fc521f3be 2024-12-10T20:15:07,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/9bd10d3dc2c94e2a9b178316945d4b5a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/9bd10d3dc2c94e2a9b178316945d4b5a 2024-12-10T20:15:07,033 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/9bd10d3dc2c94e2a9b178316945d4b5a, entries=150, sequenceid=253, filesize=11.9 K 2024-12-10T20:15:07,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/5b658b1c442d45cab8932b78c3843632 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/5b658b1c442d45cab8932b78c3843632 2024-12-10T20:15:07,044 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/5b658b1c442d45cab8932b78c3843632, entries=150, sequenceid=253, filesize=11.9 K 2024-12-10T20:15:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/3271212c956648b89572900fc521f3be as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/3271212c956648b89572900fc521f3be 2024-12-10T20:15:07,055 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/3271212c956648b89572900fc521f3be, entries=150, sequenceid=253, filesize=11.9 K 2024-12-10T20:15:07,057 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 308bff2a777bdccf66ea752f9872cdd4 in 997ms, sequenceid=253, compaction requested=true 2024-12-10T20:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-10T20:15:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-10T20:15:07,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-10T20:15:07,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-12-10T20:15:07,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.1130 sec 2024-12-10T20:15:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T20:15:07,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:07,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-10T20:15:07,103 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T20:15:07,104 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:07,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T20:15:07,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T20:15:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:07,262 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/eb75711bd1644ab582977d3fd5cc01d3 is 50, key is test_row_0/A:col10/1733861706186/Put/seqid=0 2024-12-10T20:15:07,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741888_1064 (size=12301) 2024-12-10T20:15:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:07,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861767365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861767375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861767375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T20:15:07,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861767488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861767697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861767697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861767697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:07,722 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/eb75711bd1644ab582977d3fd5cc01d3 2024-12-10T20:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T20:15:07,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c348e320693d48f3aa17908bfda0f356 is 50, key is test_row_0/B:col10/1733861706186/Put/seqid=0 2024-12-10T20:15:07,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741889_1065 (size=12301) 2024-12-10T20:15:08,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861768009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861768016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861768017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,204 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c348e320693d48f3aa17908bfda0f356 2024-12-10T20:15:08,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/7e2e4de9a52a4ce891ef2bcdc8973848 is 50, key is test_row_0/C:col10/1733861706186/Put/seqid=0 2024-12-10T20:15:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T20:15:08,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741890_1066 (size=12301) 2024-12-10T20:15:08,275 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/7e2e4de9a52a4ce891ef2bcdc8973848 2024-12-10T20:15:08,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/eb75711bd1644ab582977d3fd5cc01d3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/eb75711bd1644ab582977d3fd5cc01d3 2024-12-10T20:15:08,339 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/eb75711bd1644ab582977d3fd5cc01d3, entries=150, sequenceid=272, filesize=12.0 K 2024-12-10T20:15:08,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c348e320693d48f3aa17908bfda0f356 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c348e320693d48f3aa17908bfda0f356 2024-12-10T20:15:08,354 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c348e320693d48f3aa17908bfda0f356, entries=150, sequenceid=272, filesize=12.0 K 2024-12-10T20:15:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/7e2e4de9a52a4ce891ef2bcdc8973848 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/7e2e4de9a52a4ce891ef2bcdc8973848 2024-12-10T20:15:08,369 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/7e2e4de9a52a4ce891ef2bcdc8973848, entries=150, sequenceid=272, filesize=12.0 K 2024-12-10T20:15:08,374 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 308bff2a777bdccf66ea752f9872cdd4 in 1111ms, sequenceid=272, compaction requested=true 2024-12-10T20:15:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-10T20:15:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-10T20:15:08,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-10T20:15:08,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2750 sec 2024-12-10T20:15:08,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2840 sec 2024-12-10T20:15:08,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:15:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:08,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/042b38384de74d69b6f30a47bd25a0d7 is 50, key is test_row_0/A:col10/1733861708531/Put/seqid=0 2024-12-10T20:15:08,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741891_1067 (size=12301) 2024-12-10T20:15:08,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/042b38384de74d69b6f30a47bd25a0d7 2024-12-10T20:15:08,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861768594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861768599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861768603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/adfd60a90ee54f1384a9277f65533acf is 50, key is test_row_0/B:col10/1733861708531/Put/seqid=0 2024-12-10T20:15:08,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741892_1068 (size=12301) 2024-12-10T20:15:08,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/adfd60a90ee54f1384a9277f65533acf 2024-12-10T20:15:08,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d652ddd272884688937783cce7d9ff9f is 50, key is test_row_0/C:col10/1733861708531/Put/seqid=0 2024-12-10T20:15:08,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741893_1069 (size=12301) 2024-12-10T20:15:08,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d652ddd272884688937783cce7d9ff9f 2024-12-10T20:15:08,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861768708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861768717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/042b38384de74d69b6f30a47bd25a0d7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/042b38384de74d69b6f30a47bd25a0d7 2024-12-10T20:15:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861768720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/042b38384de74d69b6f30a47bd25a0d7, entries=150, sequenceid=288, filesize=12.0 K 2024-12-10T20:15:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/adfd60a90ee54f1384a9277f65533acf as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/adfd60a90ee54f1384a9277f65533acf 2024-12-10T20:15:08,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/adfd60a90ee54f1384a9277f65533acf, entries=150, sequenceid=288, filesize=12.0 K 2024-12-10T20:15:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d652ddd272884688937783cce7d9ff9f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d652ddd272884688937783cce7d9ff9f 2024-12-10T20:15:08,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d652ddd272884688937783cce7d9ff9f, entries=150, sequenceid=288, filesize=12.0 K 2024-12-10T20:15:08,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 308bff2a777bdccf66ea752f9872cdd4 in 239ms, sequenceid=288, compaction requested=true 2024-12-10T20:15:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:08,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:15:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:08,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T20:15:08,773 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T20:15:08,773 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T20:15:08,785 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63923 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T20:15:08,785 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:08,785 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:08,785 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5291c077b4f4eada909bbd22b20922a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/003aa63ed6304117b80d5481a087822d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/9bd10d3dc2c94e2a9b178316945d4b5a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/eb75711bd1644ab582977d3fd5cc01d3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/042b38384de74d69b6f30a47bd25a0d7] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=62.4 K 2024-12-10T20:15:08,786 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61533 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T20:15:08,786 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:08,786 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:08,786 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/208fca06b9ed49ea80e4c83ff32f5cde, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/3271212c956648b89572900fc521f3be, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/7e2e4de9a52a4ce891ef2bcdc8973848, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d652ddd272884688937783cce7d9ff9f] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=60.1 K 2024-12-10T20:15:08,787 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5291c077b4f4eada909bbd22b20922a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733861703876 2024-12-10T20:15:08,788 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003aa63ed6304117b80d5481a087822d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733861704583 2024-12-10T20:15:08,788 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb8541bc4e84a2bb1a0d2cbf590c4e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733861703876 2024-12-10T20:15:08,789 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd10d3dc2c94e2a9b178316945d4b5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733861704986 2024-12-10T20:15:08,789 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb75711bd1644ab582977d3fd5cc01d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733861706180 2024-12-10T20:15:08,789 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 042b38384de74d69b6f30a47bd25a0d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733861707360 2024-12-10T20:15:08,793 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 208fca06b9ed49ea80e4c83ff32f5cde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733861704598 2024-12-10T20:15:08,794 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 3271212c956648b89572900fc521f3be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733861704986 2024-12-10T20:15:08,795 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2e4de9a52a4ce891ef2bcdc8973848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733861706180 2024-12-10T20:15:08,796 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d652ddd272884688937783cce7d9ff9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733861707360 2024-12-10T20:15:08,809 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:08,810 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/aa7c59d42fae4926a2a2f1337162b497 is 50, key is test_row_0/A:col10/1733861708531/Put/seqid=0 2024-12-10T20:15:08,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741894_1070 (size=12949) 2024-12-10T20:15:08,849 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#58 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:08,850 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/39b68626e138425db6e534953378b524 is 50, key is test_row_0/C:col10/1733861708531/Put/seqid=0 2024-12-10T20:15:08,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741895_1071 (size=12949) 2024-12-10T20:15:08,891 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/39b68626e138425db6e534953378b524 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/39b68626e138425db6e534953378b524 2024-12-10T20:15:08,921 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into 39b68626e138425db6e534953378b524(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:08,921 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:08,921 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=11, startTime=1733861708771; duration=0sec 2024-12-10T20:15:08,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:08,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:08,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T20:15:08,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:08,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:08,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c5cf77498883433b8ae24f04a14bdaa2 is 50, key is test_row_0/A:col10/1733861708586/Put/seqid=0 2024-12-10T20:15:08,941 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61533 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T20:15:08,941 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:08,942 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:08,942 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e10512b8e7a54972b99497625b691058, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/d1edc5193e2845aeacc5f6980218cfde, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/5b658b1c442d45cab8932b78c3843632, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c348e320693d48f3aa17908bfda0f356, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/adfd60a90ee54f1384a9277f65533acf] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=60.1 K 2024-12-10T20:15:08,944 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e10512b8e7a54972b99497625b691058, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733861703876 2024-12-10T20:15:08,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741896_1072 (size=14741) 2024-12-10T20:15:08,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c5cf77498883433b8ae24f04a14bdaa2 2024-12-10T20:15:08,948 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d1edc5193e2845aeacc5f6980218cfde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733861704598 2024-12-10T20:15:08,955 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b658b1c442d45cab8932b78c3843632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733861704986 2024-12-10T20:15:08,956 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c348e320693d48f3aa17908bfda0f356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733861706180 2024-12-10T20:15:08,959 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting adfd60a90ee54f1384a9277f65533acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733861707360 2024-12-10T20:15:08,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c047fdad07374470acf1564985ba725f is 50, key is test_row_0/B:col10/1733861708586/Put/seqid=0 2024-12-10T20:15:08,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861768987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861768988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861768994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741897_1073 (size=12301) 2024-12-10T20:15:09,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c047fdad07374470acf1564985ba725f 2024-12-10T20:15:09,028 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#61 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:09,029 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/7f073ba7ea0c44299eddffbbfb8a6856 is 50, key is test_row_0/B:col10/1733861708531/Put/seqid=0 2024-12-10T20:15:09,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/add7962f2fdf4792846271b825be25ac is 50, key is test_row_0/C:col10/1733861708586/Put/seqid=0 2024-12-10T20:15:09,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741899_1075 (size=12301) 2024-12-10T20:15:09,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741898_1074 (size=12949) 2024-12-10T20:15:09,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861769097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861769107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861769107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T20:15:09,247 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:09,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-10T20:15:09,251 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:09,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:09,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:09,280 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/aa7c59d42fae4926a2a2f1337162b497 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/aa7c59d42fae4926a2a2f1337162b497 2024-12-10T20:15:09,301 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into aa7c59d42fae4926a2a2f1337162b497(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:09,301 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:09,301 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=11, startTime=1733861708770; duration=0sec 2024-12-10T20:15:09,301 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:09,302 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861769309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861769317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861769317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:09,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:09,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:09,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/add7962f2fdf4792846271b825be25ac 2024-12-10T20:15:09,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c5cf77498883433b8ae24f04a14bdaa2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5cf77498883433b8ae24f04a14bdaa2 2024-12-10T20:15:09,546 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/7f073ba7ea0c44299eddffbbfb8a6856 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7f073ba7ea0c44299eddffbbfb8a6856 2024-12-10T20:15:09,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5cf77498883433b8ae24f04a14bdaa2, entries=200, sequenceid=306, filesize=14.4 K 2024-12-10T20:15:09,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c047fdad07374470acf1564985ba725f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c047fdad07374470acf1564985ba725f 2024-12-10T20:15:09,570 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 7f073ba7ea0c44299eddffbbfb8a6856(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:09,570 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:09,571 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=11, startTime=1733861708770; duration=0sec 2024-12-10T20:15:09,571 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:09,571 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:09,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c047fdad07374470acf1564985ba725f, entries=150, sequenceid=306, filesize=12.0 K 2024-12-10T20:15:09,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/add7962f2fdf4792846271b825be25ac as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/add7962f2fdf4792846271b825be25ac 2024-12-10T20:15:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:09,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:09,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:09,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/add7962f2fdf4792846271b825be25ac, entries=150, sequenceid=306, filesize=12.0 K 2024-12-10T20:15:09,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 308bff2a777bdccf66ea752f9872cdd4 in 667ms, sequenceid=306, compaction requested=false 2024-12-10T20:15:09,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:09,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T20:15:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:09,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:09,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/497782d3de9f43ad9d682c542d354157 is 50, key is test_row_0/A:col10/1733861708992/Put/seqid=0 2024-12-10T20:15:09,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861769669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861769670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741900_1076 (size=14741) 2024-12-10T20:15:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861769675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861769789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861769789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861769791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:09,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:09,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:09,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:09,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861770001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861770009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861770008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/497782d3de9f43ad9d682c542d354157 2024-12-10T20:15:10,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/a306dbfa22b34d60944ae3b72d2b195c is 50, key is test_row_0/B:col10/1733861708992/Put/seqid=0 2024-12-10T20:15:10,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741901_1077 (size=12301) 2024-12-10T20:15:10,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/a306dbfa22b34d60944ae3b72d2b195c 2024-12-10T20:15:10,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/cb40a0177cee45fab30023f3434aeb9e is 50, key is test_row_0/C:col10/1733861708992/Put/seqid=0 2024-12-10T20:15:10,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741902_1078 (size=12301) 2024-12-10T20:15:10,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/cb40a0177cee45fab30023f3434aeb9e 2024-12-10T20:15:10,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/497782d3de9f43ad9d682c542d354157 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/497782d3de9f43ad9d682c542d354157 2024-12-10T20:15:10,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/497782d3de9f43ad9d682c542d354157, entries=200, sequenceid=324, filesize=14.4 K 2024-12-10T20:15:10,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/a306dbfa22b34d60944ae3b72d2b195c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a306dbfa22b34d60944ae3b72d2b195c 2024-12-10T20:15:10,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a306dbfa22b34d60944ae3b72d2b195c, entries=150, sequenceid=324, filesize=12.0 K 2024-12-10T20:15:10,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/cb40a0177cee45fab30023f3434aeb9e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cb40a0177cee45fab30023f3434aeb9e 2024-12-10T20:15:10,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cb40a0177cee45fab30023f3434aeb9e, entries=150, sequenceid=324, filesize=12.0 K 2024-12-10T20:15:10,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 308bff2a777bdccf66ea752f9872cdd4 in 638ms, sequenceid=324, compaction requested=true 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:10,270 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:10,270 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:10,272 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:10,272 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:10,272 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:10,272 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:10,272 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,272 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,272 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/aa7c59d42fae4926a2a2f1337162b497, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5cf77498883433b8ae24f04a14bdaa2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/497782d3de9f43ad9d682c542d354157] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=41.4 K 2024-12-10T20:15:10,272 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7f073ba7ea0c44299eddffbbfb8a6856, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c047fdad07374470acf1564985ba725f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a306dbfa22b34d60944ae3b72d2b195c] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=36.7 K 2024-12-10T20:15:10,276 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7c59d42fae4926a2a2f1337162b497, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733861707360 2024-12-10T20:15:10,276 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f073ba7ea0c44299eddffbbfb8a6856, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733861707360 2024-12-10T20:15:10,277 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c047fdad07374470acf1564985ba725f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733861708586 2024-12-10T20:15:10,277 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5cf77498883433b8ae24f04a14bdaa2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733861708570 2024-12-10T20:15:10,278 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497782d3de9f43ad9d682c542d354157, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733861708964 2024-12-10T20:15:10,278 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a306dbfa22b34d60944ae3b72d2b195c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733861708984 2024-12-10T20:15:10,306 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:10,306 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/6ab7a7c8c7eb4b5188193b124471be6c is 50, key is test_row_0/A:col10/1733861708992/Put/seqid=0 2024-12-10T20:15:10,309 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#66 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:10,310 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/3783b21c6b2542f4a09712db58534b2f is 50, key is test_row_0/B:col10/1733861708992/Put/seqid=0 2024-12-10T20:15:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:10,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:15:10,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:10,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:10,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:10,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:10,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:10,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:10,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741903_1079 (size=13051) 2024-12-10T20:15:10,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:10,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:10,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ea4d4f0f21ae451c9966df5f131c744f is 50, key is test_row_0/A:col10/1733861710320/Put/seqid=0 2024-12-10T20:15:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861770373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861770377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861770377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,394 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/6ab7a7c8c7eb4b5188193b124471be6c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/6ab7a7c8c7eb4b5188193b124471be6c 2024-12-10T20:15:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:10,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741904_1080 (size=13051) 2024-12-10T20:15:10,423 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into 6ab7a7c8c7eb4b5188193b124471be6c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:10,423 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:10,423 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=13, startTime=1733861710270; duration=0sec 2024-12-10T20:15:10,423 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:10,423 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:10,424 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:10,432 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:10,432 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:10,432 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,432 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/39b68626e138425db6e534953378b524, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/add7962f2fdf4792846271b825be25ac, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cb40a0177cee45fab30023f3434aeb9e] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=36.7 K 2024-12-10T20:15:10,444 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b68626e138425db6e534953378b524, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733861707360 2024-12-10T20:15:10,446 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting add7962f2fdf4792846271b825be25ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733861708586 2024-12-10T20:15:10,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741905_1081 (size=14741) 2024-12-10T20:15:10,451 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb40a0177cee45fab30023f3434aeb9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733861708984 2024-12-10T20:15:10,455 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/3783b21c6b2542f4a09712db58534b2f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3783b21c6b2542f4a09712db58534b2f 2024-12-10T20:15:10,477 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:10,477 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/9109a6fabd36444abdc744cd2d5235e9 is 50, key is test_row_0/C:col10/1733861708992/Put/seqid=0 2024-12-10T20:15:10,482 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 3783b21c6b2542f4a09712db58534b2f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:10,482 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:10,482 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=13, startTime=1733861710270; duration=0sec 2024-12-10T20:15:10,482 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:10,482 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861770501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861770501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861770502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741906_1082 (size=13051) 2024-12-10T20:15:10,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861770718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861770719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861770721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:10,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ea4d4f0f21ae451c9966df5f131c744f 2024-12-10T20:15:10,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:10,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:10,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/a24adea3977a4df8905f036b5a34613e is 50, key is test_row_0/B:col10/1733861710320/Put/seqid=0 2024-12-10T20:15:10,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741907_1083 (size=12301) 2024-12-10T20:15:10,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/a24adea3977a4df8905f036b5a34613e 2024-12-10T20:15:10,952 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/9109a6fabd36444abdc744cd2d5235e9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9109a6fabd36444abdc744cd2d5235e9 2024-12-10T20:15:10,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/097b2160f7994ae8aac94728f8473f35 is 50, key is test_row_0/C:col10/1733861710320/Put/seqid=0 2024-12-10T20:15:10,998 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into 9109a6fabd36444abdc744cd2d5235e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:10,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:10,999 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=13, startTime=1733861710270; duration=0sec 2024-12-10T20:15:10,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:10,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:11,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741908_1084 (size=12301) 2024-12-10T20:15:11,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/097b2160f7994ae8aac94728f8473f35 2024-12-10T20:15:11,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:11,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ea4d4f0f21ae451c9966df5f131c744f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea4d4f0f21ae451c9966df5f131c744f 2024-12-10T20:15:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861771027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861771030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861771030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea4d4f0f21ae451c9966df5f131c744f, entries=200, sequenceid=341, filesize=14.4 K 2024-12-10T20:15:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/a24adea3977a4df8905f036b5a34613e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a24adea3977a4df8905f036b5a34613e 2024-12-10T20:15:11,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a24adea3977a4df8905f036b5a34613e, entries=150, sequenceid=341, filesize=12.0 K 2024-12-10T20:15:11,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/097b2160f7994ae8aac94728f8473f35 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/097b2160f7994ae8aac94728f8473f35 2024-12-10T20:15:11,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/097b2160f7994ae8aac94728f8473f35, entries=150, sequenceid=341, filesize=12.0 K 2024-12-10T20:15:11,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 308bff2a777bdccf66ea752f9872cdd4 in 746ms, sequenceid=341, compaction requested=false 2024-12-10T20:15:11,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:11,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T20:15:11,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:11,184 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:15:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:11,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ea55086ac8e8445b81600b7b7d3bea71 is 50, key is test_row_0/A:col10/1733861710375/Put/seqid=0 2024-12-10T20:15:11,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741909_1085 (size=12301) 2024-12-10T20:15:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:11,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:11,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861771593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861771594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861771601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,619 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ea55086ac8e8445b81600b7b7d3bea71 2024-12-10T20:15:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/8683f56f2f2e46468b463ea0dbbc94ec is 50, key is test_row_0/B:col10/1733861710375/Put/seqid=0 2024-12-10T20:15:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741910_1086 (size=12301) 2024-12-10T20:15:11,686 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/8683f56f2f2e46468b463ea0dbbc94ec 2024-12-10T20:15:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861771707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861771707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861771709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b0d6ccef5eb442d29637275a24f74639 is 50, key is test_row_0/C:col10/1733861710375/Put/seqid=0 2024-12-10T20:15:11,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741911_1087 (size=12301) 2024-12-10T20:15:11,770 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b0d6ccef5eb442d29637275a24f74639 2024-12-10T20:15:11,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ea55086ac8e8445b81600b7b7d3bea71 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea55086ac8e8445b81600b7b7d3bea71 2024-12-10T20:15:11,864 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea55086ac8e8445b81600b7b7d3bea71, entries=150, sequenceid=359, filesize=12.0 K 2024-12-10T20:15:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/8683f56f2f2e46468b463ea0dbbc94ec as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8683f56f2f2e46468b463ea0dbbc94ec 2024-12-10T20:15:11,883 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8683f56f2f2e46468b463ea0dbbc94ec, entries=150, sequenceid=359, filesize=12.0 K 2024-12-10T20:15:11,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b0d6ccef5eb442d29637275a24f74639 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b0d6ccef5eb442d29637275a24f74639 2024-12-10T20:15:11,894 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b0d6ccef5eb442d29637275a24f74639, entries=150, sequenceid=359, filesize=12.0 K 2024-12-10T20:15:11,896 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 308bff2a777bdccf66ea752f9872cdd4 in 713ms, sequenceid=359, compaction requested=true 2024-12-10T20:15:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-10T20:15:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-10T20:15:11,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-10T20:15:11,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6440 sec 2024-12-10T20:15:11,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.6540 sec 2024-12-10T20:15:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:11,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T20:15:11,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:11,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:11,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:11,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:11,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:11,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:11,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ccc995f85cf7453fbcec07977f84212d is 50, key is test_row_0/A:col10/1733861711591/Put/seqid=0 2024-12-10T20:15:11,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741912_1088 (size=14741) 2024-12-10T20:15:11,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ccc995f85cf7453fbcec07977f84212d 2024-12-10T20:15:12,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861772001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861772007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861772008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/3e694b7f5b354aacaa0092c5a0b73112 is 50, key is test_row_0/B:col10/1733861711591/Put/seqid=0 2024-12-10T20:15:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741913_1089 (size=12301) 2024-12-10T20:15:12,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/3e694b7f5b354aacaa0092c5a0b73112 2024-12-10T20:15:12,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/08a7b188bcb3440abb8ba1022d38a88e is 50, key is test_row_0/C:col10/1733861711591/Put/seqid=0 2024-12-10T20:15:12,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741914_1090 (size=12301) 2024-12-10T20:15:12,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/08a7b188bcb3440abb8ba1022d38a88e 2024-12-10T20:15:12,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/ccc995f85cf7453fbcec07977f84212d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ccc995f85cf7453fbcec07977f84212d 2024-12-10T20:15:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861772118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861772121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861772118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ccc995f85cf7453fbcec07977f84212d, entries=200, sequenceid=377, filesize=14.4 K 2024-12-10T20:15:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/3e694b7f5b354aacaa0092c5a0b73112 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3e694b7f5b354aacaa0092c5a0b73112 2024-12-10T20:15:12,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3e694b7f5b354aacaa0092c5a0b73112, entries=150, sequenceid=377, filesize=12.0 K 2024-12-10T20:15:12,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/08a7b188bcb3440abb8ba1022d38a88e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/08a7b188bcb3440abb8ba1022d38a88e 2024-12-10T20:15:12,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/08a7b188bcb3440abb8ba1022d38a88e, entries=150, sequenceid=377, filesize=12.0 K 2024-12-10T20:15:12,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 308bff2a777bdccf66ea752f9872cdd4 in 259ms, sequenceid=377, compaction requested=true 2024-12-10T20:15:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:12,179 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:12,182 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:12,182 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:12,182 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:12,183 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/6ab7a7c8c7eb4b5188193b124471be6c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea4d4f0f21ae451c9966df5f131c744f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea55086ac8e8445b81600b7b7d3bea71, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ccc995f85cf7453fbcec07977f84212d] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=53.5 K 2024-12-10T20:15:12,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:12,183 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab7a7c8c7eb4b5188193b124471be6c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733861708984 2024-12-10T20:15:12,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:12,184 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:12,184 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea4d4f0f21ae451c9966df5f131c744f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733861709664 2024-12-10T20:15:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:12,184 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea55086ac8e8445b81600b7b7d3bea71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733861710364 2024-12-10T20:15:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:12,186 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc995f85cf7453fbcec07977f84212d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733861711584 2024-12-10T20:15:12,188 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:12,188 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:12,189 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:12,189 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3783b21c6b2542f4a09712db58534b2f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a24adea3977a4df8905f036b5a34613e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8683f56f2f2e46468b463ea0dbbc94ec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3e694b7f5b354aacaa0092c5a0b73112] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=48.8 K 2024-12-10T20:15:12,190 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 3783b21c6b2542f4a09712db58534b2f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733861708984 2024-12-10T20:15:12,192 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a24adea3977a4df8905f036b5a34613e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733861709667 2024-12-10T20:15:12,194 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8683f56f2f2e46468b463ea0dbbc94ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733861710364 2024-12-10T20:15:12,196 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e694b7f5b354aacaa0092c5a0b73112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733861711584 2024-12-10T20:15:12,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:12,211 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:12,211 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f9efb469bcd0444b8d10d5d853ac7113 is 50, key is test_row_0/A:col10/1733861711591/Put/seqid=0 2024-12-10T20:15:12,221 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:12,222 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/128552d601ca43a7926422c8e489926e is 50, key is test_row_0/B:col10/1733861711591/Put/seqid=0 2024-12-10T20:15:12,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741915_1091 (size=13187) 2024-12-10T20:15:12,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741916_1092 (size=13187) 2024-12-10T20:15:12,269 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f9efb469bcd0444b8d10d5d853ac7113 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f9efb469bcd0444b8d10d5d853ac7113 2024-12-10T20:15:12,277 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/128552d601ca43a7926422c8e489926e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/128552d601ca43a7926422c8e489926e 2024-12-10T20:15:12,306 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into f9efb469bcd0444b8d10d5d853ac7113(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:12,306 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 128552d601ca43a7926422c8e489926e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:12,306 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:12,306 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:12,306 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=12, startTime=1733861712178; duration=0sec 2024-12-10T20:15:12,306 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=12, startTime=1733861712183; duration=0sec 2024-12-10T20:15:12,306 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:12,306 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:12,307 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:12,313 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:12,313 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:12,316 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:12,316 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:12,316 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:12,316 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9109a6fabd36444abdc744cd2d5235e9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/097b2160f7994ae8aac94728f8473f35, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b0d6ccef5eb442d29637275a24f74639, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/08a7b188bcb3440abb8ba1022d38a88e] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=48.8 K 2024-12-10T20:15:12,320 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9109a6fabd36444abdc744cd2d5235e9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733861708984 2024-12-10T20:15:12,320 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 097b2160f7994ae8aac94728f8473f35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733861709667 2024-12-10T20:15:12,321 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d6ccef5eb442d29637275a24f74639, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733861710364 2024-12-10T20:15:12,325 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08a7b188bcb3440abb8ba1022d38a88e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733861711584 2024-12-10T20:15:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:12,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T20:15:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:12,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:12,371 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:12,372 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/ad16920de0ea402c9e551417ec7235f4 is 50, key is test_row_0/C:col10/1733861711591/Put/seqid=0 2024-12-10T20:15:12,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f05901636068432e9a5eda5a7e06376f is 50, key is test_row_0/A:col10/1733861712343/Put/seqid=0 2024-12-10T20:15:12,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861772387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861772389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861772390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741917_1093 (size=13187) 2024-12-10T20:15:12,416 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/ad16920de0ea402c9e551417ec7235f4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ad16920de0ea402c9e551417ec7235f4 2024-12-10T20:15:12,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741918_1094 (size=17181) 2024-12-10T20:15:12,426 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into ad16920de0ea402c9e551417ec7235f4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:12,426 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:12,426 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=12, startTime=1733861712184; duration=0sec 2024-12-10T20:15:12,426 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:12,426 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861772497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861772497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861772509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861772707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861772717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861772717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:12,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f05901636068432e9a5eda5a7e06376f 2024-12-10T20:15:12,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/2b1fee427cff4840920e9715d59e9157 is 50, key is test_row_0/B:col10/1733861712343/Put/seqid=0 2024-12-10T20:15:12,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741919_1095 (size=12301) 2024-12-10T20:15:12,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/2b1fee427cff4840920e9715d59e9157 2024-12-10T20:15:12,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b6e22b8821d84738ae96a32d31d7663b is 50, key is test_row_0/C:col10/1733861712343/Put/seqid=0 2024-12-10T20:15:12,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741920_1096 (size=12301) 2024-12-10T20:15:12,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b6e22b8821d84738ae96a32d31d7663b 2024-12-10T20:15:12,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f05901636068432e9a5eda5a7e06376f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f05901636068432e9a5eda5a7e06376f 2024-12-10T20:15:12,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f05901636068432e9a5eda5a7e06376f, entries=250, sequenceid=395, filesize=16.8 K 2024-12-10T20:15:12,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/2b1fee427cff4840920e9715d59e9157 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/2b1fee427cff4840920e9715d59e9157 2024-12-10T20:15:12,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/2b1fee427cff4840920e9715d59e9157, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T20:15:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b6e22b8821d84738ae96a32d31d7663b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b6e22b8821d84738ae96a32d31d7663b 2024-12-10T20:15:13,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b6e22b8821d84738ae96a32d31d7663b, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T20:15:13,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 308bff2a777bdccf66ea752f9872cdd4 in 667ms, sequenceid=395, compaction requested=false 2024-12-10T20:15:13,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:13,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:15:13,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:13,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:13,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:13,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:13,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:13,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:13,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/37fef09dfc78432c8e432c96f006d261 is 50, key is test_row_0/A:col10/1733861712375/Put/seqid=0 2024-12-10T20:15:13,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861773084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861773090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741921_1097 (size=14741) 2024-12-10T20:15:13,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861773090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/37fef09dfc78432c8e432c96f006d261 2024-12-10T20:15:13,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/bf716124937d4eda800182cdab57f895 is 50, key is test_row_0/B:col10/1733861712375/Put/seqid=0 2024-12-10T20:15:13,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741922_1098 (size=12301) 2024-12-10T20:15:13,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/bf716124937d4eda800182cdab57f895 2024-12-10T20:15:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861773198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/0e975e03ef0a4bb9a4aeb8954eab7f0c is 50, key is test_row_0/C:col10/1733861712375/Put/seqid=0 2024-12-10T20:15:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861773209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861773209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741923_1099 (size=12301) 2024-12-10T20:15:13,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/0e975e03ef0a4bb9a4aeb8954eab7f0c 2024-12-10T20:15:13,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/37fef09dfc78432c8e432c96f006d261 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/37fef09dfc78432c8e432c96f006d261 2024-12-10T20:15:13,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/37fef09dfc78432c8e432c96f006d261, entries=200, sequenceid=413, filesize=14.4 K 2024-12-10T20:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/bf716124937d4eda800182cdab57f895 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/bf716124937d4eda800182cdab57f895 2024-12-10T20:15:13,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/bf716124937d4eda800182cdab57f895, entries=150, sequenceid=413, filesize=12.0 K 2024-12-10T20:15:13,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/0e975e03ef0a4bb9a4aeb8954eab7f0c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/0e975e03ef0a4bb9a4aeb8954eab7f0c 2024-12-10T20:15:13,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/0e975e03ef0a4bb9a4aeb8954eab7f0c, entries=150, sequenceid=413, filesize=12.0 K 2024-12-10T20:15:13,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 308bff2a777bdccf66ea752f9872cdd4 in 290ms, sequenceid=413, compaction requested=true 2024-12-10T20:15:13,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:13,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:13,321 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:13,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:13,321 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:13,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:13,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:13,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:13,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:13,324 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:13,324 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:13,324 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:13,324 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:13,324 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,324 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,324 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/128552d601ca43a7926422c8e489926e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/2b1fee427cff4840920e9715d59e9157, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/bf716124937d4eda800182cdab57f895] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=36.9 K 2024-12-10T20:15:13,324 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f9efb469bcd0444b8d10d5d853ac7113, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f05901636068432e9a5eda5a7e06376f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/37fef09dfc78432c8e432c96f006d261] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=44.1 K 2024-12-10T20:15:13,327 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 128552d601ca43a7926422c8e489926e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733861711584 2024-12-10T20:15:13,327 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9efb469bcd0444b8d10d5d853ac7113, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733861711584 2024-12-10T20:15:13,334 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b1fee427cff4840920e9715d59e9157, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733861712003 2024-12-10T20:15:13,334 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05901636068432e9a5eda5a7e06376f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733861711992 2024-12-10T20:15:13,335 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting bf716124937d4eda800182cdab57f895, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733861712375 2024-12-10T20:15:13,337 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37fef09dfc78432c8e432c96f006d261, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733861712375 2024-12-10T20:15:13,376 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#87 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:13,376 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/4d58833e71624f03a5387e8ff4f00d36 is 50, key is test_row_0/A:col10/1733861712375/Put/seqid=0 2024-12-10T20:15:13,382 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:13,383 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/907fb575bdde4e81a4c5a725597c2b99 is 50, key is test_row_0/B:col10/1733861712375/Put/seqid=0 2024-12-10T20:15:13,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741925_1101 (size=13289) 2024-12-10T20:15:13,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741924_1100 (size=13289) 2024-12-10T20:15:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:13,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:15:13,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:13,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:13,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:13,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:13,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:13,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:13,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/89938654b21a4a008667264418f518ba is 50, key is test_row_0/A:col10/1733861713084/Put/seqid=0 2024-12-10T20:15:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T20:15:13,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:13,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-10T20:15:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T20:15:13,441 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:13,442 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:13,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:13,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861773456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861773459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861773465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741926_1102 (size=12301) 2024-12-10T20:15:13,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/89938654b21a4a008667264418f518ba 2024-12-10T20:15:13,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/4cbbc5d968be49168d8ffde711aba149 is 50, key is test_row_0/B:col10/1733861713084/Put/seqid=0 2024-12-10T20:15:13,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741927_1103 (size=12301) 2024-12-10T20:15:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T20:15:13,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861773567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861773567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861773576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T20:15:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T20:15:13,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:13,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T20:15:13,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861773777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861773777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861773792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:13,850 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/907fb575bdde4e81a4c5a725597c2b99 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/907fb575bdde4e81a4c5a725597c2b99 2024-12-10T20:15:13,862 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/4d58833e71624f03a5387e8ff4f00d36 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/4d58833e71624f03a5387e8ff4f00d36 2024-12-10T20:15:13,895 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 907fb575bdde4e81a4c5a725597c2b99(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:13,895 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:13,895 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=13, startTime=1733861713321; duration=0sec 2024-12-10T20:15:13,896 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:13,896 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:13,896 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:13,906 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:13,906 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:13,906 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,906 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ad16920de0ea402c9e551417ec7235f4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b6e22b8821d84738ae96a32d31d7663b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/0e975e03ef0a4bb9a4aeb8954eab7f0c] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=36.9 K 2024-12-10T20:15:13,907 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ad16920de0ea402c9e551417ec7235f4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733861711584 2024-12-10T20:15:13,907 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e22b8821d84738ae96a32d31d7663b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733861712003 2024-12-10T20:15:13,908 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e975e03ef0a4bb9a4aeb8954eab7f0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733861712375 2024-12-10T20:15:13,908 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into 4d58833e71624f03a5387e8ff4f00d36(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:13,908 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:13,908 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=13, startTime=1733861713321; duration=0sec 2024-12-10T20:15:13,908 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:13,908 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:13,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T20:15:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:13,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:13,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/4cbbc5d968be49168d8ffde711aba149 2024-12-10T20:15:13,954 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:13,955 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/eb4415d2b79d40338612f531199778c9 is 50, key is test_row_0/C:col10/1733861712375/Put/seqid=0 2024-12-10T20:15:13,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/ea6189a0a96e4f82af53d1441b7cc054 is 50, key is test_row_0/C:col10/1733861713084/Put/seqid=0 2024-12-10T20:15:13,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741928_1104 (size=13289) 2024-12-10T20:15:14,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741929_1105 (size=12301) 2024-12-10T20:15:14,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/ea6189a0a96e4f82af53d1441b7cc054 2024-12-10T20:15:14,033 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/eb4415d2b79d40338612f531199778c9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/eb4415d2b79d40338612f531199778c9 2024-12-10T20:15:14,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/89938654b21a4a008667264418f518ba as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/89938654b21a4a008667264418f518ba 2024-12-10T20:15:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T20:15:14,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T20:15:14,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/89938654b21a4a008667264418f518ba, entries=150, sequenceid=430, filesize=12.0 K 2024-12-10T20:15:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:14,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:14,081 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into eb4415d2b79d40338612f531199778c9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:14,084 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:14,084 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=13, startTime=1733861713322; duration=0sec 2024-12-10T20:15:14,084 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:14,084 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:14,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/4cbbc5d968be49168d8ffde711aba149 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4cbbc5d968be49168d8ffde711aba149 2024-12-10T20:15:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861774089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4cbbc5d968be49168d8ffde711aba149, entries=150, sequenceid=430, filesize=12.0 K 2024-12-10T20:15:14,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/ea6189a0a96e4f82af53d1441b7cc054 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ea6189a0a96e4f82af53d1441b7cc054 2024-12-10T20:15:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861774108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861774109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ea6189a0a96e4f82af53d1441b7cc054, entries=150, sequenceid=430, filesize=12.0 K 2024-12-10T20:15:14,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 308bff2a777bdccf66ea752f9872cdd4 in 701ms, sequenceid=430, compaction requested=false 2024-12-10T20:15:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:14,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T20:15:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:14,238 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:15:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/04a6674a4c5a4fe79032cafcac1ba1a7 is 50, key is test_row_0/A:col10/1733861713459/Put/seqid=0 2024-12-10T20:15:14,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741930_1106 (size=12301) 2024-12-10T20:15:14,274 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/04a6674a4c5a4fe79032cafcac1ba1a7 2024-12-10T20:15:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/772be0ae725140cbbb9dd92a967c9f5c is 50, key is test_row_0/B:col10/1733861713459/Put/seqid=0 2024-12-10T20:15:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741931_1107 (size=12301) 2024-12-10T20:15:14,329 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/772be0ae725140cbbb9dd92a967c9f5c 2024-12-10T20:15:14,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/6e437a2194544a4686acd92e6b2fdfc5 is 50, key is test_row_0/C:col10/1733861713459/Put/seqid=0 2024-12-10T20:15:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741932_1108 (size=12301) 2024-12-10T20:15:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T20:15:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:14,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861774653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861774654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861774655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861774767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861774768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861774768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:14,789 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/6e437a2194544a4686acd92e6b2fdfc5 2024-12-10T20:15:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/04a6674a4c5a4fe79032cafcac1ba1a7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/04a6674a4c5a4fe79032cafcac1ba1a7 2024-12-10T20:15:14,814 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/04a6674a4c5a4fe79032cafcac1ba1a7, entries=150, sequenceid=448, filesize=12.0 K 2024-12-10T20:15:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/772be0ae725140cbbb9dd92a967c9f5c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/772be0ae725140cbbb9dd92a967c9f5c 2024-12-10T20:15:14,822 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/772be0ae725140cbbb9dd92a967c9f5c, entries=150, sequenceid=448, filesize=12.0 K 2024-12-10T20:15:14,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/6e437a2194544a4686acd92e6b2fdfc5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6e437a2194544a4686acd92e6b2fdfc5 2024-12-10T20:15:14,839 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6e437a2194544a4686acd92e6b2fdfc5, entries=150, sequenceid=448, filesize=12.0 K 2024-12-10T20:15:14,840 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 308bff2a777bdccf66ea752f9872cdd4 in 602ms, sequenceid=448, compaction requested=true 2024-12-10T20:15:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-10T20:15:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-10T20:15:14,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-10T20:15:14,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-12-10T20:15:14,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4120 sec 2024-12-10T20:15:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:14,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T20:15:14,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:14,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:14,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:14,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:14,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:14,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:15,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/93ebb83747f445109e924edd3acba502 is 50, key is test_row_0/A:col10/1733861714990/Put/seqid=0 2024-12-10T20:15:15,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861775019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861775017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861775021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741933_1109 (size=14741) 2024-12-10T20:15:15,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/93ebb83747f445109e924edd3acba502 2024-12-10T20:15:15,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e31a3712088c495596a48c2c51507ac8 is 50, key is test_row_0/B:col10/1733861714990/Put/seqid=0 2024-12-10T20:15:15,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741934_1110 (size=12301) 2024-12-10T20:15:15,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e31a3712088c495596a48c2c51507ac8 2024-12-10T20:15:15,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d72d894c0115433fb5630657b005958e is 50, key is test_row_0/C:col10/1733861714990/Put/seqid=0 2024-12-10T20:15:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861775137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861775137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861775137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741935_1111 (size=12301) 2024-12-10T20:15:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47522 deadline: 1733861775162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,169 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18306 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47536 deadline: 1733861775347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47580 deadline: 1733861775348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1733861775348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1733861775458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:15,469 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 308bff2a777bdccf66ea752f9872cdd4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18329 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:15:15,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d72d894c0115433fb5630657b005958e 2024-12-10T20:15:15,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/93ebb83747f445109e924edd3acba502 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/93ebb83747f445109e924edd3acba502 2024-12-10T20:15:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T20:15:15,588 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:15,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:15,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/93ebb83747f445109e924edd3acba502, entries=200, sequenceid=468, filesize=14.4 K 2024-12-10T20:15:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-10T20:15:15,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/e31a3712088c495596a48c2c51507ac8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e31a3712088c495596a48c2c51507ac8 2024-12-10T20:15:15,595 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T20:15:15,596 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:15,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:15,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e31a3712088c495596a48c2c51507ac8, entries=150, sequenceid=468, filesize=12.0 K 2024-12-10T20:15:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/d72d894c0115433fb5630657b005958e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d72d894c0115433fb5630657b005958e 2024-12-10T20:15:15,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d72d894c0115433fb5630657b005958e, entries=150, sequenceid=468, filesize=12.0 K 2024-12-10T20:15:15,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 308bff2a777bdccf66ea752f9872cdd4 in 634ms, sequenceid=468, compaction requested=true 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:15,624 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 308bff2a777bdccf66ea752f9872cdd4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:15,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:15:15,626 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:15,627 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/A is initiating minor compaction (all files) 2024-12-10T20:15:15,627 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/A in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,627 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/4d58833e71624f03a5387e8ff4f00d36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/89938654b21a4a008667264418f518ba, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/04a6674a4c5a4fe79032cafcac1ba1a7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/93ebb83747f445109e924edd3acba502] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=51.4 K 2024-12-10T20:15:15,628 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d58833e71624f03a5387e8ff4f00d36, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733861712375 2024-12-10T20:15:15,628 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89938654b21a4a008667264418f518ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861713084 2024-12-10T20:15:15,628 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:15,629 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a6674a4c5a4fe79032cafcac1ba1a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733861713454 2024-12-10T20:15:15,629 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ebb83747f445109e924edd3acba502, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733861714653 2024-12-10T20:15:15,632 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:15,632 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/B is initiating minor compaction (all files) 2024-12-10T20:15:15,632 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/B in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,632 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/907fb575bdde4e81a4c5a725597c2b99, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4cbbc5d968be49168d8ffde711aba149, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/772be0ae725140cbbb9dd92a967c9f5c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e31a3712088c495596a48c2c51507ac8] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=49.0 K 2024-12-10T20:15:15,633 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 907fb575bdde4e81a4c5a725597c2b99, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733861712375 2024-12-10T20:15:15,634 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cbbc5d968be49168d8ffde711aba149, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861713084 2024-12-10T20:15:15,634 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 772be0ae725140cbbb9dd92a967c9f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733861713454 2024-12-10T20:15:15,635 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e31a3712088c495596a48c2c51507ac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733861714653 2024-12-10T20:15:15,659 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#A#compaction#99 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:15,660 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/33b433ce088b40d29d0326e2a4682802 is 50, key is test_row_0/A:col10/1733861714990/Put/seqid=0 2024-12-10T20:15:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:15,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:15:15,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:15,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:15,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:15,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:15,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:15,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:15,676 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#B#compaction#100 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:15,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f09900c3127b4de291fdd8c1ec97b7a9 is 50, key is test_row_0/A:col10/1733861715016/Put/seqid=0 2024-12-10T20:15:15,678 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/7002f3a1dc3f4dc6ae25874ee6f9215e is 50, key is test_row_0/B:col10/1733861714990/Put/seqid=0 2024-12-10T20:15:15,697 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-10T20:15:15,698 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,704 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,705 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,705 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-10T20:15:15,705 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,705 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,705 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,707 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-10T20:15:15,707 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,707 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,707 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T20:15:15,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,709 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-10T20:15:15,709 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,710 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,710 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741936_1112 (size=13425) 2024-12-10T20:15:15,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,714 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-10T20:15:15,714 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,714 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,714 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,715 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-10T20:15:15,715 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,715 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,715 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,722 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-10T20:15:15,722 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:15,722 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,722 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:15,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:15,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741938_1114 (size=13425) 2024-12-10T20:15:15,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741937_1113 (size=14741) 2024-12-10T20:15:15,743 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/33b433ce088b40d29d0326e2a4682802 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/33b433ce088b40d29d0326e2a4682802 2024-12-10T20:15:15,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f09900c3127b4de291fdd8c1ec97b7a9 2024-12-10T20:15:15,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T20:15:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:15,754 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/A of 308bff2a777bdccf66ea752f9872cdd4 into 33b433ce088b40d29d0326e2a4682802(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:15,755 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:15,755 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/A, priority=12, startTime=1733861715624; duration=0sec 2024-12-10T20:15:15,755 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:15,755 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:A 2024-12-10T20:15:15,755 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:15,762 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:15,762 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 308bff2a777bdccf66ea752f9872cdd4/C is initiating minor compaction (all files) 2024-12-10T20:15:15,763 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 308bff2a777bdccf66ea752f9872cdd4/C in TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,763 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/eb4415d2b79d40338612f531199778c9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ea6189a0a96e4f82af53d1441b7cc054, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6e437a2194544a4686acd92e6b2fdfc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d72d894c0115433fb5630657b005958e] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp, totalSize=49.0 K 2024-12-10T20:15:15,763 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb4415d2b79d40338612f531199778c9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733861712375 2024-12-10T20:15:15,764 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea6189a0a96e4f82af53d1441b7cc054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861713084 2024-12-10T20:15:15,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c2fd294f0e104972b78b3fa4e06c2e3a is 50, key is test_row_0/B:col10/1733861715016/Put/seqid=0 2024-12-10T20:15:15,765 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e437a2194544a4686acd92e6b2fdfc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733861713454 2024-12-10T20:15:15,766 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting d72d894c0115433fb5630657b005958e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733861714653 2024-12-10T20:15:15,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741939_1115 (size=12301) 2024-12-10T20:15:15,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c2fd294f0e104972b78b3fa4e06c2e3a 2024-12-10T20:15:15,791 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 308bff2a777bdccf66ea752f9872cdd4#C#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:15,792 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/dbd0ccad3141460f974ab602607a6394 is 50, key is test_row_0/C:col10/1733861714990/Put/seqid=0 2024-12-10T20:15:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741940_1116 (size=13425) 2024-12-10T20:15:15,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b5b69819b1fb4f369e47d5838acc852b is 50, key is test_row_0/C:col10/1733861715016/Put/seqid=0 2024-12-10T20:15:15,812 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/dbd0ccad3141460f974ab602607a6394 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/dbd0ccad3141460f974ab602607a6394 2024-12-10T20:15:15,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741941_1117 (size=12301) 2024-12-10T20:15:15,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b5b69819b1fb4f369e47d5838acc852b 2024-12-10T20:15:15,826 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/C of 308bff2a777bdccf66ea752f9872cdd4 into dbd0ccad3141460f974ab602607a6394(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:15,826 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:15,826 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/C, priority=12, startTime=1733861715624; duration=0sec 2024-12-10T20:15:15,826 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:15,826 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:C 2024-12-10T20:15:15,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/f09900c3127b4de291fdd8c1ec97b7a9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f09900c3127b4de291fdd8c1ec97b7a9 2024-12-10T20:15:15,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f09900c3127b4de291fdd8c1ec97b7a9, entries=200, sequenceid=481, filesize=14.4 K 2024-12-10T20:15:15,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/c2fd294f0e104972b78b3fa4e06c2e3a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c2fd294f0e104972b78b3fa4e06c2e3a 2024-12-10T20:15:15,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c2fd294f0e104972b78b3fa4e06c2e3a, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T20:15:15,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/b5b69819b1fb4f369e47d5838acc852b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b5b69819b1fb4f369e47d5838acc852b 2024-12-10T20:15:15,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T20:15:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. as already flushing 2024-12-10T20:15:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:15,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T20:15:15,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b5b69819b1fb4f369e47d5838acc852b, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T20:15:15,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=46.96 KB/48090 for 308bff2a777bdccf66ea752f9872cdd4 in 262ms, sequenceid=481, compaction requested=false 2024-12-10T20:15:15,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:16,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T20:15:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:16,072 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:15:16,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/8f63f5f9fc8949089398772ea0745e4f is 50, key is test_row_0/A:col10/1733861715704/Put/seqid=0 2024-12-10T20:15:16,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741942_1118 (size=9857) 2024-12-10T20:15:16,159 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/7002f3a1dc3f4dc6ae25874ee6f9215e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7002f3a1dc3f4dc6ae25874ee6f9215e 2024-12-10T20:15:16,173 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 308bff2a777bdccf66ea752f9872cdd4/B of 308bff2a777bdccf66ea752f9872cdd4 into 7002f3a1dc3f4dc6ae25874ee6f9215e(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:16,173 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:16,173 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., storeName=308bff2a777bdccf66ea752f9872cdd4/B, priority=12, startTime=1733861715624; duration=0sec 2024-12-10T20:15:16,173 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:16,173 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 308bff2a777bdccf66ea752f9872cdd4:B 2024-12-10T20:15:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T20:15:16,521 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/8f63f5f9fc8949089398772ea0745e4f 2024-12-10T20:15:16,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/8fd7047bf01347a6b48fcadac9a6cc53 is 50, key is test_row_0/B:col10/1733861715704/Put/seqid=0 2024-12-10T20:15:16,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741943_1119 (size=9857) 2024-12-10T20:15:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T20:15:16,936 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/8fd7047bf01347a6b48fcadac9a6cc53 2024-12-10T20:15:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/aa445ab19eda40d7a848ff2eedb4048a is 50, key is test_row_0/C:col10/1733861715704/Put/seqid=0 2024-12-10T20:15:16,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741944_1120 (size=9857) 2024-12-10T20:15:17,351 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/aa445ab19eda40d7a848ff2eedb4048a 2024-12-10T20:15:17,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/8f63f5f9fc8949089398772ea0745e4f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/8f63f5f9fc8949089398772ea0745e4f 2024-12-10T20:15:17,361 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/8f63f5f9fc8949089398772ea0745e4f, entries=100, sequenceid=493, filesize=9.6 K 2024-12-10T20:15:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/8fd7047bf01347a6b48fcadac9a6cc53 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8fd7047bf01347a6b48fcadac9a6cc53 2024-12-10T20:15:17,367 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8fd7047bf01347a6b48fcadac9a6cc53, entries=100, sequenceid=493, filesize=9.6 K 2024-12-10T20:15:17,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/aa445ab19eda40d7a848ff2eedb4048a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/aa445ab19eda40d7a848ff2eedb4048a 2024-12-10T20:15:17,373 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/aa445ab19eda40d7a848ff2eedb4048a, entries=100, sequenceid=493, filesize=9.6 K 2024-12-10T20:15:17,374 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 308bff2a777bdccf66ea752f9872cdd4 in 1302ms, sequenceid=493, compaction requested=true 2024-12-10T20:15:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 308bff2a777bdccf66ea752f9872cdd4: 2024-12-10T20:15:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-10T20:15:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-10T20:15:17,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-10T20:15:17,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-12-10T20:15:17,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7880 sec 2024-12-10T20:15:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T20:15:17,757 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:17,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:15:25,248 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-10T20:15:25,248 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:25,248 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:25,248 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:25,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:25,498 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-10T20:15:25,499 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:25,499 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:25,499 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:25,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2922 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2836 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1339 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4016 rows 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1341 2024-12-10T20:15:25,499 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4022 rows 2024-12-10T20:15:25,499 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:15:25,500 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:15:25,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:25,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:25,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:25,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T20:15:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:25,513 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861725512"}]},"ts":"1733861725512"} 2024-12-10T20:15:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T20:15:25,516 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T20:15:25,516 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T20:15:25,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:15:25,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, UNASSIGN}] 2024-12-10T20:15:25,524 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, UNASSIGN 2024-12-10T20:15:25,526 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=308bff2a777bdccf66ea752f9872cdd4, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:25,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, UNASSIGN because future has completed 2024-12-10T20:15:25,530 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:15:25,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:15:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T20:15:25,688 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:25,688 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:15:25,689 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 308bff2a777bdccf66ea752f9872cdd4, disabling compactions & flushes 2024-12-10T20:15:25,689 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:25,689 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:25,689 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. after waiting 0 ms 2024-12-10T20:15:25,689 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:25,689 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 308bff2a777bdccf66ea752f9872cdd4 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-10T20:15:25,690 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=A 2024-12-10T20:15:25,690 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:25,690 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=B 2024-12-10T20:15:25,690 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:25,690 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 308bff2a777bdccf66ea752f9872cdd4, store=C 2024-12-10T20:15:25,690 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:25,694 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c4f7d25fbabe49d9a291c9d1b2e59912 is 50, key is test_row_1/A:col10/1733861725497/Put/seqid=0 2024-12-10T20:15:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741945_1121 (size=9857) 2024-12-10T20:15:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T20:15:26,100 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c4f7d25fbabe49d9a291c9d1b2e59912 2024-12-10T20:15:26,109 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/0d1ab95d4aeb424b9f87b088cfbda1dc is 50, key is test_row_1/B:col10/1733861725497/Put/seqid=0 2024-12-10T20:15:26,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741946_1122 (size=9857) 2024-12-10T20:15:26,114 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/0d1ab95d4aeb424b9f87b088cfbda1dc 2024-12-10T20:15:26,121 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/e86a4e70d4b54f5f91ffa0bd7909966e is 50, key is test_row_1/C:col10/1733861725497/Put/seqid=0 2024-12-10T20:15:26,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741947_1123 (size=9857) 2024-12-10T20:15:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T20:15:26,526 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/e86a4e70d4b54f5f91ffa0bd7909966e 2024-12-10T20:15:26,534 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/A/c4f7d25fbabe49d9a291c9d1b2e59912 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c4f7d25fbabe49d9a291c9d1b2e59912 2024-12-10T20:15:26,539 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c4f7d25fbabe49d9a291c9d1b2e59912, entries=100, sequenceid=499, filesize=9.6 K 2024-12-10T20:15:26,540 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/B/0d1ab95d4aeb424b9f87b088cfbda1dc as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/0d1ab95d4aeb424b9f87b088cfbda1dc 2024-12-10T20:15:26,547 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/0d1ab95d4aeb424b9f87b088cfbda1dc, entries=100, sequenceid=499, filesize=9.6 K 2024-12-10T20:15:26,548 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/.tmp/C/e86a4e70d4b54f5f91ffa0bd7909966e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/e86a4e70d4b54f5f91ffa0bd7909966e 2024-12-10T20:15:26,552 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/e86a4e70d4b54f5f91ffa0bd7909966e, entries=100, sequenceid=499, filesize=9.6 K 2024-12-10T20:15:26,553 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 308bff2a777bdccf66ea752f9872cdd4 in 864ms, sequenceid=499, compaction requested=true 2024-12-10T20:15:26,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/de50ebe607fb4231a2047ecc332aa770, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/2778c595e25e466691b67a2001f0383a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/940b503128504135b48911052a680a3d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/572f9d421ee04e86a376196e191a3437, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/dab7ee9f6efd4c33ae2bd6ea2930c650, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/02508363e56f40faa7b676173408e1ef, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f470cb5477ec48eb835a86060ea8dbea, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/817f676b169d408586cf76585602d261, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/06fac488dd7e4da69f04c48884b3a201, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ce66aff9f2d6447f9ee8100cff8ce1f0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ac00d222de24487e8e66d2d87e8c6f2b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/81b55e9f84934e4897d93136daed98af, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/71006d0c0a8e4aa08a29653b9a13d100, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c8510dd1824a4a749bd4132b536098ad, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5291c077b4f4eada909bbd22b20922a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/003aa63ed6304117b80d5481a087822d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/9bd10d3dc2c94e2a9b178316945d4b5a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/eb75711bd1644ab582977d3fd5cc01d3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/aa7c59d42fae4926a2a2f1337162b497, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/042b38384de74d69b6f30a47bd25a0d7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5cf77498883433b8ae24f04a14bdaa2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/497782d3de9f43ad9d682c542d354157, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/6ab7a7c8c7eb4b5188193b124471be6c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea4d4f0f21ae451c9966df5f131c744f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea55086ac8e8445b81600b7b7d3bea71, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ccc995f85cf7453fbcec07977f84212d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f9efb469bcd0444b8d10d5d853ac7113, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f05901636068432e9a5eda5a7e06376f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/37fef09dfc78432c8e432c96f006d261, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/4d58833e71624f03a5387e8ff4f00d36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/89938654b21a4a008667264418f518ba, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/04a6674a4c5a4fe79032cafcac1ba1a7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/93ebb83747f445109e924edd3acba502] to archive 2024-12-10T20:15:26,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:15:26,563 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/817f676b169d408586cf76585602d261 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/817f676b169d408586cf76585602d261 2024-12-10T20:15:26,563 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/2778c595e25e466691b67a2001f0383a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/2778c595e25e466691b67a2001f0383a 2024-12-10T20:15:26,563 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/940b503128504135b48911052a680a3d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/940b503128504135b48911052a680a3d 2024-12-10T20:15:26,564 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/02508363e56f40faa7b676173408e1ef to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/02508363e56f40faa7b676173408e1ef 2024-12-10T20:15:26,564 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/572f9d421ee04e86a376196e191a3437 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/572f9d421ee04e86a376196e191a3437 2024-12-10T20:15:26,564 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/dab7ee9f6efd4c33ae2bd6ea2930c650 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/dab7ee9f6efd4c33ae2bd6ea2930c650 2024-12-10T20:15:26,564 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f470cb5477ec48eb835a86060ea8dbea to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f470cb5477ec48eb835a86060ea8dbea 2024-12-10T20:15:26,564 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/de50ebe607fb4231a2047ecc332aa770 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/de50ebe607fb4231a2047ecc332aa770 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ac00d222de24487e8e66d2d87e8c6f2b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ac00d222de24487e8e66d2d87e8c6f2b 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/71006d0c0a8e4aa08a29653b9a13d100 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/71006d0c0a8e4aa08a29653b9a13d100 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c8510dd1824a4a749bd4132b536098ad to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c8510dd1824a4a749bd4132b536098ad 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5291c077b4f4eada909bbd22b20922a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5291c077b4f4eada909bbd22b20922a 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ce66aff9f2d6447f9ee8100cff8ce1f0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ce66aff9f2d6447f9ee8100cff8ce1f0 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/003aa63ed6304117b80d5481a087822d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/003aa63ed6304117b80d5481a087822d 2024-12-10T20:15:26,566 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/81b55e9f84934e4897d93136daed98af to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/81b55e9f84934e4897d93136daed98af 2024-12-10T20:15:26,568 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/06fac488dd7e4da69f04c48884b3a201 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/06fac488dd7e4da69f04c48884b3a201 2024-12-10T20:15:26,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/9bd10d3dc2c94e2a9b178316945d4b5a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/9bd10d3dc2c94e2a9b178316945d4b5a 2024-12-10T20:15:26,569 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/aa7c59d42fae4926a2a2f1337162b497 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/aa7c59d42fae4926a2a2f1337162b497 2024-12-10T20:15:26,569 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5cf77498883433b8ae24f04a14bdaa2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c5cf77498883433b8ae24f04a14bdaa2 2024-12-10T20:15:26,569 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/042b38384de74d69b6f30a47bd25a0d7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/042b38384de74d69b6f30a47bd25a0d7 2024-12-10T20:15:26,569 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/eb75711bd1644ab582977d3fd5cc01d3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/eb75711bd1644ab582977d3fd5cc01d3 2024-12-10T20:15:26,569 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/497782d3de9f43ad9d682c542d354157 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/497782d3de9f43ad9d682c542d354157 2024-12-10T20:15:26,570 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/6ab7a7c8c7eb4b5188193b124471be6c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/6ab7a7c8c7eb4b5188193b124471be6c 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea55086ac8e8445b81600b7b7d3bea71 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea55086ac8e8445b81600b7b7d3bea71 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ccc995f85cf7453fbcec07977f84212d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ccc995f85cf7453fbcec07977f84212d 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/37fef09dfc78432c8e432c96f006d261 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/37fef09dfc78432c8e432c96f006d261 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f9efb469bcd0444b8d10d5d853ac7113 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f9efb469bcd0444b8d10d5d853ac7113 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea4d4f0f21ae451c9966df5f131c744f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/ea4d4f0f21ae451c9966df5f131c744f 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f05901636068432e9a5eda5a7e06376f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f05901636068432e9a5eda5a7e06376f 2024-12-10T20:15:26,571 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/89938654b21a4a008667264418f518ba to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/89938654b21a4a008667264418f518ba 2024-12-10T20:15:26,572 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/4d58833e71624f03a5387e8ff4f00d36 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/4d58833e71624f03a5387e8ff4f00d36 2024-12-10T20:15:26,572 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/04a6674a4c5a4fe79032cafcac1ba1a7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/04a6674a4c5a4fe79032cafcac1ba1a7 2024-12-10T20:15:26,572 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/93ebb83747f445109e924edd3acba502 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/93ebb83747f445109e924edd3acba502 2024-12-10T20:15:26,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/11e1764df4af430f9f37e74364fc1f9e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/dfe9017e5abe4c5695c9f968af136517, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/6692100815ec4bc194b3a4bf20a14ba7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/15da92bd72a740a3996337d579df4c0a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/35b52394185c43279417fb8d70da2cb7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e329ccce0ad441299635b92b952f4e8b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/ab9a5cb4a2b641b6b34c1f8b53fbd163, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c43f0c447d0242cfa859e5a7dc072a99, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4bf394ee7f42469d841b130e8af4be7a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/097fdcac26e4409d96327efe0a1faa08, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/512b64b25d53482c8926284a4bfee3b4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/477626c127614b9c960edf51781c9e26, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/40b6246d39af423894dd7ffa86e5b807, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e10512b8e7a54972b99497625b691058, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/1e349201a3f040cbb6c590f121f4dd14, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/d1edc5193e2845aeacc5f6980218cfde, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/5b658b1c442d45cab8932b78c3843632, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c348e320693d48f3aa17908bfda0f356, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7f073ba7ea0c44299eddffbbfb8a6856, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/adfd60a90ee54f1384a9277f65533acf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c047fdad07374470acf1564985ba725f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3783b21c6b2542f4a09712db58534b2f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a306dbfa22b34d60944ae3b72d2b195c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a24adea3977a4df8905f036b5a34613e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8683f56f2f2e46468b463ea0dbbc94ec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/128552d601ca43a7926422c8e489926e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3e694b7f5b354aacaa0092c5a0b73112, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/2b1fee427cff4840920e9715d59e9157, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/907fb575bdde4e81a4c5a725597c2b99, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/bf716124937d4eda800182cdab57f895, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4cbbc5d968be49168d8ffde711aba149, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/772be0ae725140cbbb9dd92a967c9f5c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e31a3712088c495596a48c2c51507ac8] to archive 2024-12-10T20:15:26,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:15:26,586 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/11e1764df4af430f9f37e74364fc1f9e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/11e1764df4af430f9f37e74364fc1f9e 2024-12-10T20:15:26,587 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/6692100815ec4bc194b3a4bf20a14ba7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/6692100815ec4bc194b3a4bf20a14ba7 2024-12-10T20:15:26,587 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/35b52394185c43279417fb8d70da2cb7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/35b52394185c43279417fb8d70da2cb7 2024-12-10T20:15:26,587 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c43f0c447d0242cfa859e5a7dc072a99 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c43f0c447d0242cfa859e5a7dc072a99 2024-12-10T20:15:26,587 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/ab9a5cb4a2b641b6b34c1f8b53fbd163 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/ab9a5cb4a2b641b6b34c1f8b53fbd163 2024-12-10T20:15:26,587 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e329ccce0ad441299635b92b952f4e8b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e329ccce0ad441299635b92b952f4e8b 2024-12-10T20:15:26,587 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/dfe9017e5abe4c5695c9f968af136517 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/dfe9017e5abe4c5695c9f968af136517 2024-12-10T20:15:26,588 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4bf394ee7f42469d841b130e8af4be7a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4bf394ee7f42469d841b130e8af4be7a 2024-12-10T20:15:26,588 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/512b64b25d53482c8926284a4bfee3b4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/512b64b25d53482c8926284a4bfee3b4 2024-12-10T20:15:26,589 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/15da92bd72a740a3996337d579df4c0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/15da92bd72a740a3996337d579df4c0a 2024-12-10T20:15:26,589 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/477626c127614b9c960edf51781c9e26 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/477626c127614b9c960edf51781c9e26 2024-12-10T20:15:26,589 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/097fdcac26e4409d96327efe0a1faa08 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/097fdcac26e4409d96327efe0a1faa08 2024-12-10T20:15:26,589 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e10512b8e7a54972b99497625b691058 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e10512b8e7a54972b99497625b691058 2024-12-10T20:15:26,589 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/40b6246d39af423894dd7ffa86e5b807 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/40b6246d39af423894dd7ffa86e5b807 2024-12-10T20:15:26,590 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/1e349201a3f040cbb6c590f121f4dd14 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/1e349201a3f040cbb6c590f121f4dd14 2024-12-10T20:15:26,590 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/d1edc5193e2845aeacc5f6980218cfde to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/d1edc5193e2845aeacc5f6980218cfde 2024-12-10T20:15:26,591 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/5b658b1c442d45cab8932b78c3843632 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/5b658b1c442d45cab8932b78c3843632 2024-12-10T20:15:26,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c348e320693d48f3aa17908bfda0f356 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c348e320693d48f3aa17908bfda0f356 2024-12-10T20:15:26,591 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7f073ba7ea0c44299eddffbbfb8a6856 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7f073ba7ea0c44299eddffbbfb8a6856 2024-12-10T20:15:26,591 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c047fdad07374470acf1564985ba725f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c047fdad07374470acf1564985ba725f 2024-12-10T20:15:26,592 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/adfd60a90ee54f1384a9277f65533acf to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/adfd60a90ee54f1384a9277f65533acf 2024-12-10T20:15:26,592 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a306dbfa22b34d60944ae3b72d2b195c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a306dbfa22b34d60944ae3b72d2b195c 2024-12-10T20:15:26,592 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3783b21c6b2542f4a09712db58534b2f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3783b21c6b2542f4a09712db58534b2f 2024-12-10T20:15:26,594 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a24adea3977a4df8905f036b5a34613e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/a24adea3977a4df8905f036b5a34613e 2024-12-10T20:15:26,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/128552d601ca43a7926422c8e489926e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/128552d601ca43a7926422c8e489926e 2024-12-10T20:15:26,595 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3e694b7f5b354aacaa0092c5a0b73112 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/3e694b7f5b354aacaa0092c5a0b73112 2024-12-10T20:15:26,595 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/907fb575bdde4e81a4c5a725597c2b99 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/907fb575bdde4e81a4c5a725597c2b99 2024-12-10T20:15:26,595 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/2b1fee427cff4840920e9715d59e9157 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/2b1fee427cff4840920e9715d59e9157 2024-12-10T20:15:26,595 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8683f56f2f2e46468b463ea0dbbc94ec to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8683f56f2f2e46468b463ea0dbbc94ec 2024-12-10T20:15:26,595 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/bf716124937d4eda800182cdab57f895 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/bf716124937d4eda800182cdab57f895 2024-12-10T20:15:26,595 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4cbbc5d968be49168d8ffde711aba149 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/4cbbc5d968be49168d8ffde711aba149 2024-12-10T20:15:26,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e31a3712088c495596a48c2c51507ac8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/e31a3712088c495596a48c2c51507ac8 2024-12-10T20:15:26,597 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/772be0ae725140cbbb9dd92a967c9f5c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/772be0ae725140cbbb9dd92a967c9f5c 2024-12-10T20:15:26,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9f5825ffbc284b7284333f0a2dfaadae, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/83bb9a2f35e74e16ae002da9e4f47677, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/907e9b1e07d949918ac4c69d953df6c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/77834a5dd28643baa31cd7a4b44784ca, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6f8276f6e02042fcb61b1b9e11f2f517, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/4dbcb9adaab147b493a0a8b5aa445a64, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/bfd600d14f674a1b82699012c75781e1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/33843b73e837426eadd765c7a098fe02, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c8517c4f92ce4a4faec2d3aae420bafd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/a3f4c9baae614158ad112d6e525b82c6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/171f8f9b64d841cebad3f6e7ef1a421c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d6ddcd9fa308431691b18ddc47c565cf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/858d9e974b8448599642c71fc40336b3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c39fe1aacfaf4ac69bf1f96123270dc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/208fca06b9ed49ea80e4c83ff32f5cde, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/3271212c956648b89572900fc521f3be, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/7e2e4de9a52a4ce891ef2bcdc8973848, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/39b68626e138425db6e534953378b524, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d652ddd272884688937783cce7d9ff9f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/add7962f2fdf4792846271b825be25ac, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9109a6fabd36444abdc744cd2d5235e9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cb40a0177cee45fab30023f3434aeb9e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/097b2160f7994ae8aac94728f8473f35, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b0d6ccef5eb442d29637275a24f74639, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ad16920de0ea402c9e551417ec7235f4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/08a7b188bcb3440abb8ba1022d38a88e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b6e22b8821d84738ae96a32d31d7663b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/eb4415d2b79d40338612f531199778c9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/0e975e03ef0a4bb9a4aeb8954eab7f0c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ea6189a0a96e4f82af53d1441b7cc054, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6e437a2194544a4686acd92e6b2fdfc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d72d894c0115433fb5630657b005958e] to archive 2024-12-10T20:15:26,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:15:26,603 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/907e9b1e07d949918ac4c69d953df6c5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/907e9b1e07d949918ac4c69d953df6c5 2024-12-10T20:15:26,603 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9f5825ffbc284b7284333f0a2dfaadae to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9f5825ffbc284b7284333f0a2dfaadae 2024-12-10T20:15:26,603 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/83bb9a2f35e74e16ae002da9e4f47677 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/83bb9a2f35e74e16ae002da9e4f47677 2024-12-10T20:15:26,605 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/77834a5dd28643baa31cd7a4b44784ca to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/77834a5dd28643baa31cd7a4b44784ca 2024-12-10T20:15:26,605 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6f8276f6e02042fcb61b1b9e11f2f517 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6f8276f6e02042fcb61b1b9e11f2f517 2024-12-10T20:15:26,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/4dbcb9adaab147b493a0a8b5aa445a64 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/4dbcb9adaab147b493a0a8b5aa445a64 2024-12-10T20:15:26,609 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/bfd600d14f674a1b82699012c75781e1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/bfd600d14f674a1b82699012c75781e1 2024-12-10T20:15:26,609 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/33843b73e837426eadd765c7a098fe02 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/33843b73e837426eadd765c7a098fe02 2024-12-10T20:15:26,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c8517c4f92ce4a4faec2d3aae420bafd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c8517c4f92ce4a4faec2d3aae420bafd 2024-12-10T20:15:26,612 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/171f8f9b64d841cebad3f6e7ef1a421c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/171f8f9b64d841cebad3f6e7ef1a421c 2024-12-10T20:15:26,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/a3f4c9baae614158ad112d6e525b82c6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/a3f4c9baae614158ad112d6e525b82c6 2024-12-10T20:15:26,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d6ddcd9fa308431691b18ddc47c565cf to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d6ddcd9fa308431691b18ddc47c565cf 2024-12-10T20:15:26,615 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/858d9e974b8448599642c71fc40336b3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/858d9e974b8448599642c71fc40336b3 2024-12-10T20:15:26,615 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cfb8541bc4e84a2bb1a0d2cbf590c4e5 2024-12-10T20:15:26,616 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/3271212c956648b89572900fc521f3be to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/3271212c956648b89572900fc521f3be 2024-12-10T20:15:26,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/7e2e4de9a52a4ce891ef2bcdc8973848 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/7e2e4de9a52a4ce891ef2bcdc8973848 2024-12-10T20:15:26,616 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/208fca06b9ed49ea80e4c83ff32f5cde to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/208fca06b9ed49ea80e4c83ff32f5cde 2024-12-10T20:15:26,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/39b68626e138425db6e534953378b524 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/39b68626e138425db6e534953378b524 2024-12-10T20:15:26,617 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c39fe1aacfaf4ac69bf1f96123270dc5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/c39fe1aacfaf4ac69bf1f96123270dc5 2024-12-10T20:15:26,623 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d652ddd272884688937783cce7d9ff9f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d652ddd272884688937783cce7d9ff9f 2024-12-10T20:15:26,623 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9109a6fabd36444abdc744cd2d5235e9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/9109a6fabd36444abdc744cd2d5235e9 2024-12-10T20:15:26,624 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/add7962f2fdf4792846271b825be25ac to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/add7962f2fdf4792846271b825be25ac 2024-12-10T20:15:26,624 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ad16920de0ea402c9e551417ec7235f4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ad16920de0ea402c9e551417ec7235f4 2024-12-10T20:15:26,624 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cb40a0177cee45fab30023f3434aeb9e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/cb40a0177cee45fab30023f3434aeb9e 2024-12-10T20:15:26,624 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/097b2160f7994ae8aac94728f8473f35 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/097b2160f7994ae8aac94728f8473f35 2024-12-10T20:15:26,624 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b0d6ccef5eb442d29637275a24f74639 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b0d6ccef5eb442d29637275a24f74639 2024-12-10T20:15:26,624 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/08a7b188bcb3440abb8ba1022d38a88e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/08a7b188bcb3440abb8ba1022d38a88e 2024-12-10T20:15:26,625 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/eb4415d2b79d40338612f531199778c9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/eb4415d2b79d40338612f531199778c9 2024-12-10T20:15:26,625 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b6e22b8821d84738ae96a32d31d7663b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b6e22b8821d84738ae96a32d31d7663b 2024-12-10T20:15:26,625 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/0e975e03ef0a4bb9a4aeb8954eab7f0c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/0e975e03ef0a4bb9a4aeb8954eab7f0c 2024-12-10T20:15:26,626 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6e437a2194544a4686acd92e6b2fdfc5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/6e437a2194544a4686acd92e6b2fdfc5 2024-12-10T20:15:26,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ea6189a0a96e4f82af53d1441b7cc054 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/ea6189a0a96e4f82af53d1441b7cc054 2024-12-10T20:15:26,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d72d894c0115433fb5630657b005958e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/d72d894c0115433fb5630657b005958e 2024-12-10T20:15:26,632 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/recovered.edits/502.seqid, newMaxSeqId=502, maxSeqId=1 2024-12-10T20:15:26,634 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. 2024-12-10T20:15:26,634 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 308bff2a777bdccf66ea752f9872cdd4: Waiting for close lock at 1733861725689Running coprocessor pre-close hooks at 1733861725689Disabling compacts and flushes for region at 1733861725689Disabling writes for close at 1733861725689Obtaining lock to block concurrent updates at 1733861725689Preparing flush snapshotting stores in 308bff2a777bdccf66ea752f9872cdd4 at 1733861725689Finished memstore snapshotting TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733861725690 (+1 ms)Flushing stores of TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4. at 1733861725690Flushing 308bff2a777bdccf66ea752f9872cdd4/A: creating writer at 1733861725691 (+1 ms)Flushing 308bff2a777bdccf66ea752f9872cdd4/A: appending metadata at 1733861725694 (+3 ms)Flushing 308bff2a777bdccf66ea752f9872cdd4/A: closing flushed file at 1733861725694Flushing 308bff2a777bdccf66ea752f9872cdd4/B: creating writer at 1733861726105 (+411 ms)Flushing 308bff2a777bdccf66ea752f9872cdd4/B: appending metadata at 1733861726108 (+3 ms)Flushing 308bff2a777bdccf66ea752f9872cdd4/B: closing flushed file at 1733861726108Flushing 308bff2a777bdccf66ea752f9872cdd4/C: creating writer at 1733861726118 (+10 ms)Flushing 308bff2a777bdccf66ea752f9872cdd4/C: appending metadata at 1733861726120 (+2 ms)Flushing 308bff2a777bdccf66ea752f9872cdd4/C: closing flushed file at 1733861726120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cdd79a1: reopening flushed file at 1733861726533 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e6d90ca: reopening flushed file at 1733861726540 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d56f7bc: reopening flushed file at 1733861726547 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 308bff2a777bdccf66ea752f9872cdd4 in 864ms, sequenceid=499, compaction requested=true at 1733861726553 (+6 ms)Writing region close event to WAL at 1733861726628 (+75 ms)Running coprocessor post-close hooks at 1733861726633 (+5 ms)Closed at 1733861726633 2024-12-10T20:15:26,636 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:26,637 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=308bff2a777bdccf66ea752f9872cdd4, regionState=CLOSED 2024-12-10T20:15:26,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:15:26,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-10T20:15:26,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 308bff2a777bdccf66ea752f9872cdd4, server=03a2ae042c95,38133,1733861689839 in 1.1100 sec 2024-12-10T20:15:26,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-10T20:15:26,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=308bff2a777bdccf66ea752f9872cdd4, UNASSIGN in 1.1180 sec 2024-12-10T20:15:26,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-10T20:15:26,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1260 sec 2024-12-10T20:15:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T20:15:26,648 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861726647"}]},"ts":"1733861726647"} 2024-12-10T20:15:26,649 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T20:15:26,649 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T20:15:26,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1440 sec 2024-12-10T20:15:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T20:15:27,658 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:27,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T20:15:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,671 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-10T20:15:27,674 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,679 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:27,684 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/recovered.edits] 2024-12-10T20:15:27,688 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c4f7d25fbabe49d9a291c9d1b2e59912 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/c4f7d25fbabe49d9a291c9d1b2e59912 2024-12-10T20:15:27,688 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f09900c3127b4de291fdd8c1ec97b7a9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/f09900c3127b4de291fdd8c1ec97b7a9 2024-12-10T20:15:27,688 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/33b433ce088b40d29d0326e2a4682802 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/33b433ce088b40d29d0326e2a4682802 2024-12-10T20:15:27,688 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/8f63f5f9fc8949089398772ea0745e4f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/A/8f63f5f9fc8949089398772ea0745e4f 2024-12-10T20:15:27,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7002f3a1dc3f4dc6ae25874ee6f9215e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/7002f3a1dc3f4dc6ae25874ee6f9215e 2024-12-10T20:15:27,692 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8fd7047bf01347a6b48fcadac9a6cc53 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/8fd7047bf01347a6b48fcadac9a6cc53 2024-12-10T20:15:27,692 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c2fd294f0e104972b78b3fa4e06c2e3a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/c2fd294f0e104972b78b3fa4e06c2e3a 2024-12-10T20:15:27,692 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/0d1ab95d4aeb424b9f87b088cfbda1dc to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/B/0d1ab95d4aeb424b9f87b088cfbda1dc 2024-12-10T20:15:27,695 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/aa445ab19eda40d7a848ff2eedb4048a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/aa445ab19eda40d7a848ff2eedb4048a 2024-12-10T20:15:27,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/dbd0ccad3141460f974ab602607a6394 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/dbd0ccad3141460f974ab602607a6394 2024-12-10T20:15:27,696 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/e86a4e70d4b54f5f91ffa0bd7909966e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/e86a4e70d4b54f5f91ffa0bd7909966e 2024-12-10T20:15:27,696 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b5b69819b1fb4f369e47d5838acc852b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/C/b5b69819b1fb4f369e47d5838acc852b 2024-12-10T20:15:27,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/recovered.edits/502.seqid to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4/recovered.edits/502.seqid 2024-12-10T20:15:27,700 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/308bff2a777bdccf66ea752f9872cdd4 2024-12-10T20:15:27,700 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T20:15:27,704 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-10T20:15:27,712 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T20:15:27,738 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T20:15:27,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T20:15:27,740 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733861727739"}]},"ts":"9223372036854775807"} 2024-12-10T20:15:27,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T20:15:27,743 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 308bff2a777bdccf66ea752f9872cdd4, NAME => 'TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T20:15:27,744 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T20:15:27,744 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733861727744"}]},"ts":"9223372036854775807"} 2024-12-10T20:15:27,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T20:15:27,748 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-12-10T20:15:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-10T20:15:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T20:15:27,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:27,788 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=251 (was 208) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;03a2ae042c95:38133-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1060333068_22 at /127.0.0.1:44048 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 754), ProcessCount=11 (was 11), AvailableMemoryMB=249 (was 716) 2024-12-10T20:15:27,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=248 2024-12-10T20:15:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:15:27,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:15:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:27,802 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:15:27,802 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:27,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-10T20:15:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T20:15:27,803 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:15:27,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741948_1124 (size=1061) 2024-12-10T20:15:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T20:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T20:15:28,216 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:15:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741949_1125 (size=53) 2024-12-10T20:15:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T20:15:28,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:15:28,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fa895558d9876220a3ffb0096daff23e, disabling compactions & flushes 2024-12-10T20:15:28,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,629 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. after waiting 0 ms 2024-12-10T20:15:28,629 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,629 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,629 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fa895558d9876220a3ffb0096daff23e: Waiting for close lock at 1733861728628Disabling compacts and flushes for region at 1733861728628Disabling writes for close at 1733861728629 (+1 ms)Writing region close event to WAL at 1733861728629Closed at 1733861728629 2024-12-10T20:15:28,631 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:15:28,632 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733861728631"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733861728631"}]},"ts":"1733861728631"} 2024-12-10T20:15:28,636 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:15:28,637 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:15:28,637 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861728637"}]},"ts":"1733861728637"} 2024-12-10T20:15:28,640 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T20:15:28,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, ASSIGN}] 2024-12-10T20:15:28,642 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, ASSIGN 2024-12-10T20:15:28,643 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, ASSIGN; state=OFFLINE, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=false 2024-12-10T20:15:28,794 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:28,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, ASSIGN because future has completed 2024-12-10T20:15:28,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:15:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T20:15:28,958 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,958 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:15:28,959 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,959 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:15:28,959 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,959 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,961 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,963 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:15:28,963 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa895558d9876220a3ffb0096daff23e columnFamilyName A 2024-12-10T20:15:28,964 DEBUG [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:28,964 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(327): Store=fa895558d9876220a3ffb0096daff23e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:15:28,964 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,965 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:15:28,965 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa895558d9876220a3ffb0096daff23e columnFamilyName B 2024-12-10T20:15:28,966 DEBUG [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:28,966 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(327): Store=fa895558d9876220a3ffb0096daff23e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:15:28,966 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,967 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:15:28,967 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa895558d9876220a3ffb0096daff23e columnFamilyName C 2024-12-10T20:15:28,967 DEBUG [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:28,968 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(327): Store=fa895558d9876220a3ffb0096daff23e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:15:28,968 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,968 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,969 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,969 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,970 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,970 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,970 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:15:28,971 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,973 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:15:28,974 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened fa895558d9876220a3ffb0096daff23e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59468558, jitterRate=-0.11384943127632141}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:15:28,974 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:28,974 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for fa895558d9876220a3ffb0096daff23e: Running coprocessor pre-open hook at 1733861728960Writing region info on filesystem at 1733861728960Initializing all the Stores at 1733861728961 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861728961Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861728961Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861728961Cleaning up temporary data from old regions at 1733861728970 (+9 ms)Running coprocessor post-open hooks at 1733861728974 (+4 ms)Region opened successfully at 1733861728974 2024-12-10T20:15:28,975 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., pid=34, masterSystemTime=1733861728948 2024-12-10T20:15:28,977 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,977 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:28,977 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:28,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:15:28,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-10T20:15:28,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 in 184 msec 2024-12-10T20:15:28,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-10T20:15:28,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, ASSIGN in 342 msec 2024-12-10T20:15:28,984 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:15:28,984 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861728984"}]},"ts":"1733861728984"} 2024-12-10T20:15:28,986 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T20:15:28,987 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:15:28,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-12-10T20:15:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T20:15:29,948 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:29,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35053261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:29,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:29,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:29,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28aa5b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:29,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9753ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:29,958 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:29,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:29,963 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:15:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:15:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@590288b4 2024-12-10T20:15:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:15:29,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:15:29,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:15:29,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:15:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:30,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741950_1126 (size=1097) 2024-12-10T20:15:30,417 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T20:15:30,417 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T20:15:30,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:15:30,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, REOPEN/MOVE}] 2024-12-10T20:15:30,432 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, REOPEN/MOVE 2024-12-10T20:15:30,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:30,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, REOPEN/MOVE because future has completed 2024-12-10T20:15:30,435 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:15:30,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:15:30,589 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,589 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:15:30,589 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing fa895558d9876220a3ffb0096daff23e, disabling compactions & flushes 2024-12-10T20:15:30,589 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,589 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,589 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. after waiting 0 ms 2024-12-10T20:15:30,590 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,598 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T20:15:30,599 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,599 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for fa895558d9876220a3ffb0096daff23e: Waiting for close lock at 1733861730589Running coprocessor pre-close hooks at 1733861730589Disabling compacts and flushes for region at 1733861730589Disabling writes for close at 1733861730589Writing region close event to WAL at 1733861730591 (+2 ms)Running coprocessor post-close hooks at 1733861730599 (+8 ms)Closed at 1733861730599 2024-12-10T20:15:30,600 WARN [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: fa895558d9876220a3ffb0096daff23e to self. 2024-12-10T20:15:30,602 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,602 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=CLOSED 2024-12-10T20:15:30,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:15:30,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-10T20:15:30,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 in 170 msec 2024-12-10T20:15:30,608 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, REOPEN/MOVE; state=CLOSED, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=true 2024-12-10T20:15:30,759 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:30,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, REOPEN/MOVE because future has completed 2024-12-10T20:15:30,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:15:30,924 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,924 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:15:30,925 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,925 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:15:30,925 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,925 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,928 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,929 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:15:30,935 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa895558d9876220a3ffb0096daff23e columnFamilyName A 2024-12-10T20:15:30,937 DEBUG [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:30,938 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(327): Store=fa895558d9876220a3ffb0096daff23e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:15:30,938 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,939 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:15:30,939 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa895558d9876220a3ffb0096daff23e columnFamilyName B 2024-12-10T20:15:30,939 DEBUG [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:30,939 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(327): Store=fa895558d9876220a3ffb0096daff23e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:15:30,939 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,940 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:15:30,940 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa895558d9876220a3ffb0096daff23e columnFamilyName C 2024-12-10T20:15:30,940 DEBUG [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:30,941 INFO [StoreOpener-fa895558d9876220a3ffb0096daff23e-1 {}] regionserver.HStore(327): Store=fa895558d9876220a3ffb0096daff23e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:15:30,941 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,941 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,941 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,942 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,943 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,943 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,944 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:15:30,945 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,946 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened fa895558d9876220a3ffb0096daff23e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60126341, jitterRate=-0.10404770076274872}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:15:30,946 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:30,947 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for fa895558d9876220a3ffb0096daff23e: Running coprocessor pre-open hook at 1733861730925Writing region info on filesystem at 1733861730925Initializing all the Stores at 1733861730926 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861730927 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861730927Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861730927Cleaning up temporary data from old regions at 1733861730943 (+16 ms)Running coprocessor post-open hooks at 1733861730946 (+3 ms)Region opened successfully at 1733861730947 (+1 ms) 2024-12-10T20:15:30,948 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., pid=39, masterSystemTime=1733861730916 2024-12-10T20:15:30,950 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,950 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:30,950 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=OPEN, openSeqNum=5, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:30,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:15:30,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-10T20:15:30,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 in 190 msec 2024-12-10T20:15:30,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-10T20:15:30,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, REOPEN/MOVE in 525 msec 2024-12-10T20:15:30,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-10T20:15:30,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-12-10T20:15:30,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-12-10T20:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-10T20:15:30,965 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:30,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa768d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,967 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6ab570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,968 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9f6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc6bab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,970 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,972 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e87d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef6e97c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,974 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ead7095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec8bc0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60093174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8038dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fc8d88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4776c78e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29975e76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399339a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6723353c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db5debe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8444a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aece006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea40902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,988 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dca7d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe9245f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f81573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef014ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5990680b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:15:30,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:15:30,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:15:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:15:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:15:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20dd6a28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:15:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:15:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:30,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:15:30,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:15:30,996 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,996 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,996 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,996 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,996 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,996 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,997 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:30,998 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:31,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,000 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:15:31,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:15:31,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:15:31,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-10T20:15:31,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:31,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:15:31,003 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:31,004 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:31,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:15:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:31,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:15:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:31,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:31,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861791039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861731033, max timestamp=1733861731039Num uniques 859; 2024-12-10T20:15:31,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:31,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861731033, max timestamp=1733861731039Num uniques 859; 2024-12-10T20:15:31,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:31,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861731033, max timestamp=1733861731039Num uniques 859; 2024-12-10T20:15:31,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:31,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121085f40c7685224ad48234645bdb82fd6b_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861731015/Put/seqid=0 2024-12-10T20:15:31,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741951_1127 (size=9714) 2024-12-10T20:15:31,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:31,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121085f40c7685224ad48234645bdb82fd6b_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121085f40c7685224ad48234645bdb82fd6b_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:31,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/cb2b0b5f2c7143e0a65f5e1443581368, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:31,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/cb2b0b5f2c7143e0a65f5e1443581368 is 175, key is test_row_0/A:col10/1733861731015/Put/seqid=0 2024-12-10T20:15:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861791072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861791094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861791098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861791096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741952_1128 (size=22361) 2024-12-10T20:15:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:31,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861791147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861791207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861791207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861791207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861791207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:31,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861791357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861791417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861791417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861791417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861791417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/cb2b0b5f2c7143e0a65f5e1443581368 2024-12-10T20:15:31,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6f9a7f37bae94164be67a0843cb933d4 is 50, key is test_row_0/B:col10/1733861731015/Put/seqid=0 2024-12-10T20:15:31,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741953_1129 (size=9657) 2024-12-10T20:15:31,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:31,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861791667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861791727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861791727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861791727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861791727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:31,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:31,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:31,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:31,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:31,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6f9a7f37bae94164be67a0843cb933d4 2024-12-10T20:15:31,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/a1306fe8b7a14c94833bb7b2d980da78 is 50, key is test_row_0/C:col10/1733861731015/Put/seqid=0 2024-12-10T20:15:31,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741954_1130 (size=9657) 2024-12-10T20:15:31,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/a1306fe8b7a14c94833bb7b2d980da78 2024-12-10T20:15:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/cb2b0b5f2c7143e0a65f5e1443581368 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368 2024-12-10T20:15:31,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368, entries=100, sequenceid=14, filesize=21.8 K 2024-12-10T20:15:31,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6f9a7f37bae94164be67a0843cb933d4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6f9a7f37bae94164be67a0843cb933d4 2024-12-10T20:15:31,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6f9a7f37bae94164be67a0843cb933d4, entries=100, sequenceid=14, filesize=9.4 K 2024-12-10T20:15:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/a1306fe8b7a14c94833bb7b2d980da78 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a1306fe8b7a14c94833bb7b2d980da78 2024-12-10T20:15:31,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a1306fe8b7a14c94833bb7b2d980da78, entries=100, sequenceid=14, filesize=9.4 K 2024-12-10T20:15:31,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fa895558d9876220a3ffb0096daff23e in 980ms, sequenceid=14, compaction requested=false 2024-12-10T20:15:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T20:15:31,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:32,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T20:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:32,076 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T20:15:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:32,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:32,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105ac157c5cc684b7bbc8cd86d71647118_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861731095/Put/seqid=0 2024-12-10T20:15:32,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741955_1131 (size=12154) 2024-12-10T20:15:32,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:32,098 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105ac157c5cc684b7bbc8cd86d71647118_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105ac157c5cc684b7bbc8cd86d71647118_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f2b23ecd1e1f4bfc93e75d88561c0800, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:32,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f2b23ecd1e1f4bfc93e75d88561c0800 is 175, key is test_row_0/A:col10/1733861731095/Put/seqid=0 2024-12-10T20:15:32,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741956_1132 (size=30955) 2024-12-10T20:15:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:32,164 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:15:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:32,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:32,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861792214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861792237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861792237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861792237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861792237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861792327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,505 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f2b23ecd1e1f4bfc93e75d88561c0800 2024-12-10T20:15:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/91741b57199f491a8e370a4839d6e61e is 50, key is test_row_0/B:col10/1733861731095/Put/seqid=0 2024-12-10T20:15:32,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741957_1133 (size=12001) 2024-12-10T20:15:32,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861792537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861792847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:32,918 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/91741b57199f491a8e370a4839d6e61e 2024-12-10T20:15:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/2b89c8b5fdb04ed7b95167932a794450 is 50, key is test_row_0/C:col10/1733861731095/Put/seqid=0 2024-12-10T20:15:32,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741958_1134 (size=12001) 2024-12-10T20:15:33,151 INFO [master/03a2ae042c95:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T20:15:33,151 INFO [master/03a2ae042c95:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T20:15:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:33,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861793257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861793257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861793258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,260 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,260 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:33,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861793258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:33,260 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:33,263 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:33,330 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/2b89c8b5fdb04ed7b95167932a794450 2024-12-10T20:15:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f2b23ecd1e1f4bfc93e75d88561c0800 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800 2024-12-10T20:15:33,341 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800, entries=150, sequenceid=51, filesize=30.2 K 2024-12-10T20:15:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/91741b57199f491a8e370a4839d6e61e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/91741b57199f491a8e370a4839d6e61e 2024-12-10T20:15:33,348 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/91741b57199f491a8e370a4839d6e61e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T20:15:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/2b89c8b5fdb04ed7b95167932a794450 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2b89c8b5fdb04ed7b95167932a794450 2024-12-10T20:15:33,356 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2b89c8b5fdb04ed7b95167932a794450, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T20:15:33,359 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fa895558d9876220a3ffb0096daff23e in 1284ms, sequenceid=51, compaction requested=false 2024-12-10T20:15:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-10T20:15:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-10T20:15:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:33,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:15:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:33,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-10T20:15:33,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3580 sec 2024-12-10T20:15:33,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.3650 sec 2024-12-10T20:15:33,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e86bda3c33df4674b9575692f71c0fe1_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:33,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741959_1135 (size=14594) 2024-12-10T20:15:33,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861733413, max timestamp=1733861733438Num uniques 859; 2024-12-10T20:15:33,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861733413, max timestamp=1733861733438Num uniques 859; 2024-12-10T20:15:33,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861733413, max timestamp=1733861733438Num uniques 859; 2024-12-10T20:15:33,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:15:33,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:15:33,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:15:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861793501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861793607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:33,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:33,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e86bda3c33df4674b9575692f71c0fe1_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e86bda3c33df4674b9575692f71c0fe1_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:33,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/7037b3c3335343e4a241d83b1b10d09a, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:33,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/7037b3c3335343e4a241d83b1b10d09a is 175, key is test_row_0/A:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:33,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741960_1136 (size=39549) 2024-12-10T20:15:33,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861793816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861794127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:34,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/7037b3c3335343e4a241d83b1b10d09a 2024-12-10T20:15:34,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/3a82ba85c03141bcb6218e9ec9d75d08 is 50, key is test_row_0/B:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:34,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741961_1137 (size=12001) 2024-12-10T20:15:34,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/3a82ba85c03141bcb6218e9ec9d75d08 2024-12-10T20:15:34,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/e334b32c942c4fd1bbcb49dd775b5fba is 50, key is test_row_0/C:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:34,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741962_1138 (size=12001) 2024-12-10T20:15:34,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/e334b32c942c4fd1bbcb49dd775b5fba 2024-12-10T20:15:34,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/7037b3c3335343e4a241d83b1b10d09a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a 2024-12-10T20:15:34,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a, entries=200, sequenceid=62, filesize=38.6 K 2024-12-10T20:15:34,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/3a82ba85c03141bcb6218e9ec9d75d08 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/3a82ba85c03141bcb6218e9ec9d75d08 2024-12-10T20:15:34,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/3a82ba85c03141bcb6218e9ec9d75d08, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T20:15:34,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/e334b32c942c4fd1bbcb49dd775b5fba as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e334b32c942c4fd1bbcb49dd775b5fba 2024-12-10T20:15:34,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e334b32c942c4fd1bbcb49dd775b5fba, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T20:15:34,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fa895558d9876220a3ffb0096daff23e in 903ms, sequenceid=62, compaction requested=true 2024-12-10T20:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:34,264 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:34,265 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:34,266 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:34,266 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:34,266 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:34,266 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=90.7 K 2024-12-10T20:15:34,266 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:34,266 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a] 2024-12-10T20:15:34,267 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:34,267 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:34,267 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:34,267 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb2b0b5f2c7143e0a65f5e1443581368, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733861731013 2024-12-10T20:15:34,267 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6f9a7f37bae94164be67a0843cb933d4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/91741b57199f491a8e370a4839d6e61e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/3a82ba85c03141bcb6218e9ec9d75d08] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=32.9 K 2024-12-10T20:15:34,268 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f9a7f37bae94164be67a0843cb933d4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733861731013 2024-12-10T20:15:34,268 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b23ecd1e1f4bfc93e75d88561c0800, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733861731059 2024-12-10T20:15:34,268 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 91741b57199f491a8e370a4839d6e61e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733861731059 2024-12-10T20:15:34,268 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7037b3c3335343e4a241d83b1b10d09a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861732181 2024-12-10T20:15:34,268 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a82ba85c03141bcb6218e9ec9d75d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861732181 2024-12-10T20:15:34,276 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:34,277 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:34,278 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/e34e95bdcb7c4996befef1c5a9c3bf75 is 50, key is test_row_0/B:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:34,279 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121067ac23aaaf214594a1c184061592b408_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:34,283 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121067ac23aaaf214594a1c184061592b408_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:34,283 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121067ac23aaaf214594a1c184061592b408_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:34,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741963_1139 (size=12104) 2024-12-10T20:15:34,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741964_1140 (size=4469) 2024-12-10T20:15:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:34,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-10T20:15:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:34,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:34,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:34,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121043ee902bc34e4384b5f593ee94c818c2_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861734640/Put/seqid=0 2024-12-10T20:15:34,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741965_1141 (size=14594) 2024-12-10T20:15:34,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861794681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:34,690 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#121 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:34,690 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f960a9d32e5b47998bdaf3d0b8226cdc is 175, key is test_row_0/A:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:34,690 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/e34e95bdcb7c4996befef1c5a9c3bf75 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e34e95bdcb7c4996befef1c5a9c3bf75 2024-12-10T20:15:34,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741966_1142 (size=31058) 2024-12-10T20:15:34,699 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into e34e95bdcb7c4996befef1c5a9c3bf75(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:34,699 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:34,699 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=13, startTime=1733861734265; duration=0sec 2024-12-10T20:15:34,700 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:34,700 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:34,700 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:34,702 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f960a9d32e5b47998bdaf3d0b8226cdc as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f960a9d32e5b47998bdaf3d0b8226cdc 2024-12-10T20:15:34,702 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:34,702 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:34,702 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:34,703 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a1306fe8b7a14c94833bb7b2d980da78, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2b89c8b5fdb04ed7b95167932a794450, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e334b32c942c4fd1bbcb49dd775b5fba] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=32.9 K 2024-12-10T20:15:34,703 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a1306fe8b7a14c94833bb7b2d980da78, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733861731013 2024-12-10T20:15:34,704 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b89c8b5fdb04ed7b95167932a794450, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733861731059 2024-12-10T20:15:34,706 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e334b32c942c4fd1bbcb49dd775b5fba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861732181 2024-12-10T20:15:34,711 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into f960a9d32e5b47998bdaf3d0b8226cdc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:34,711 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:34,711 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=13, startTime=1733861734264; duration=0sec 2024-12-10T20:15:34,711 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:34,711 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:34,716 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:34,717 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7b165492d7c64c918f172e9265d602f3 is 50, key is test_row_0/C:col10/1733861732181/Put/seqid=0 2024-12-10T20:15:34,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741967_1143 (size=12104) 2024-12-10T20:15:34,729 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7b165492d7c64c918f172e9265d602f3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7b165492d7c64c918f172e9265d602f3 2024-12-10T20:15:34,735 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into 7b165492d7c64c918f172e9265d602f3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:34,735 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:34,735 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=13, startTime=1733861734265; duration=0sec 2024-12-10T20:15:34,735 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:34,735 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861794797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861795006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121043ee902bc34e4384b5f593ee94c818c2_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121043ee902bc34e4384b5f593ee94c818c2_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:35,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/4e3f01c9814049e698e03895889d0c36, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:35,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/4e3f01c9814049e698e03895889d0c36 is 175, key is test_row_0/A:col10/1733861734640/Put/seqid=0 2024-12-10T20:15:35,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741968_1144 (size=39549) 2024-12-10T20:15:35,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/4e3f01c9814049e698e03895889d0c36 2024-12-10T20:15:35,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/5b56f9f66e39449b85cdd47c6116cf83 is 50, key is test_row_0/B:col10/1733861734640/Put/seqid=0 2024-12-10T20:15:35,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741969_1145 (size=12001) 2024-12-10T20:15:35,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/5b56f9f66e39449b85cdd47c6116cf83 2024-12-10T20:15:35,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/da8b101d5ea54df39cff213fa597fcab is 50, key is test_row_0/C:col10/1733861734640/Put/seqid=0 2024-12-10T20:15:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741970_1146 (size=12001) 2024-12-10T20:15:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T20:15:35,177 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:35,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-10T20:15:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T20:15:35,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:35,181 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:35,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:35,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861795267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,268 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861795277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,278 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T20:15:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861795287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861795287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,289 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,290 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861795317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:35,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T20:15:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:35,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:35,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T20:15:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:35,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:35,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/da8b101d5ea54df39cff213fa597fcab 2024-12-10T20:15:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T20:15:35,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/4e3f01c9814049e698e03895889d0c36 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36 2024-12-10T20:15:35,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36, entries=200, sequenceid=97, filesize=38.6 K 2024-12-10T20:15:35,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/5b56f9f66e39449b85cdd47c6116cf83 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5b56f9f66e39449b85cdd47c6116cf83 2024-12-10T20:15:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5b56f9f66e39449b85cdd47c6116cf83, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T20:15:35,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/da8b101d5ea54df39cff213fa597fcab as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/da8b101d5ea54df39cff213fa597fcab 2024-12-10T20:15:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/da8b101d5ea54df39cff213fa597fcab, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T20:15:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for fa895558d9876220a3ffb0096daff23e in 878ms, sequenceid=97, compaction requested=false 2024-12-10T20:15:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T20:15:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:35,640 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:15:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109cce29007cbc492ebd19d830192dde8d_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_1/A:col10/1733861734679/Put/seqid=0 2024-12-10T20:15:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741971_1147 (size=9714) 2024-12-10T20:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T20:15:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:35,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:35,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861735878, max timestamp=1733861735904Num uniques 859; 2024-12-10T20:15:35,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861735878, max timestamp=1733861735904Num uniques 859; 2024-12-10T20:15:35,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861735878, max timestamp=1733861735904Num uniques 859; 2024-12-10T20:15:35,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:35,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:35,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:35,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861795965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:36,061 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109cce29007cbc492ebd19d830192dde8d_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109cce29007cbc492ebd19d830192dde8d_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/8ad031c3690d496ebdf874569684d135, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:36,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/8ad031c3690d496ebdf874569684d135 is 175, key is test_row_1/A:col10/1733861734679/Put/seqid=0 2024-12-10T20:15:36,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741972_1148 (size=22361) 2024-12-10T20:15:36,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861796077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:36,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861796287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T20:15:36,471 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/8ad031c3690d496ebdf874569684d135 2024-12-10T20:15:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/7c8ec1b86dc2498f9e8a17f31ad0313d is 50, key is test_row_1/B:col10/1733861734679/Put/seqid=0 2024-12-10T20:15:36,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741973_1149 (size=9657) 2024-12-10T20:15:36,483 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/7c8ec1b86dc2498f9e8a17f31ad0313d 2024-12-10T20:15:36,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/0d3a6b978b614fb7bfced8dd57a7691e is 50, key is test_row_1/C:col10/1733861734679/Put/seqid=0 2024-12-10T20:15:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741974_1150 (size=9657) 2024-12-10T20:15:36,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861796597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:36,895 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/0d3a6b978b614fb7bfced8dd57a7691e 2024-12-10T20:15:36,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/8ad031c3690d496ebdf874569684d135 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135 2024-12-10T20:15:36,905 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135, entries=100, sequenceid=110, filesize=21.8 K 2024-12-10T20:15:36,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/7c8ec1b86dc2498f9e8a17f31ad0313d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7c8ec1b86dc2498f9e8a17f31ad0313d 2024-12-10T20:15:36,914 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7c8ec1b86dc2498f9e8a17f31ad0313d, entries=100, sequenceid=110, filesize=9.4 K 2024-12-10T20:15:36,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/0d3a6b978b614fb7bfced8dd57a7691e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/0d3a6b978b614fb7bfced8dd57a7691e 2024-12-10T20:15:36,931 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/0d3a6b978b614fb7bfced8dd57a7691e, entries=100, sequenceid=110, filesize=9.4 K 2024-12-10T20:15:36,932 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fa895558d9876220a3ffb0096daff23e in 1292ms, sequenceid=110, compaction requested=true 2024-12-10T20:15:36,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:36,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:36,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-10T20:15:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-10T20:15:36,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-10T20:15:36,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7550 sec 2024-12-10T20:15:36,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7610 sec 2024-12-10T20:15:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:37,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T20:15:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:37,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102eba0865e61d4ffd83aac6a450778749_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861797145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741975_1151 (size=14794) 2024-12-10T20:15:37,152 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102eba0865e61d4ffd83aac6a450778749_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102eba0865e61d4ffd83aac6a450778749_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:37,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/0a9b3fd9e4ac4bcab478b83608bc35cd, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:37,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/0a9b3fd9e4ac4bcab478b83608bc35cd is 175, key is test_row_0/A:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741976_1152 (size=39749) 2024-12-10T20:15:37,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/0a9b3fd9e4ac4bcab478b83608bc35cd 2024-12-10T20:15:37,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/87fcc0d3a79f4680b6babcf25122e44b is 50, key is test_row_0/B:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741977_1153 (size=12151) 2024-12-10T20:15:37,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/87fcc0d3a79f4680b6babcf25122e44b 2024-12-10T20:15:37,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/6b200d8f835a48839d2bcc136acdb293 is 50, key is test_row_0/C:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861797257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:37,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741978_1154 (size=12151) 2024-12-10T20:15:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T20:15:37,327 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:37,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-10T20:15:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:37,330 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:37,331 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:37,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:37,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861797467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:37,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:37,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:37,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:37,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:37,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:37,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/6b200d8f835a48839d2bcc136acdb293 2024-12-10T20:15:37,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/0a9b3fd9e4ac4bcab478b83608bc35cd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd 2024-12-10T20:15:37,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd, entries=200, sequenceid=147, filesize=38.8 K 2024-12-10T20:15:37,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/87fcc0d3a79f4680b6babcf25122e44b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/87fcc0d3a79f4680b6babcf25122e44b 2024-12-10T20:15:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/87fcc0d3a79f4680b6babcf25122e44b, entries=150, sequenceid=147, filesize=11.9 K 2024-12-10T20:15:37,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/6b200d8f835a48839d2bcc136acdb293 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6b200d8f835a48839d2bcc136acdb293 2024-12-10T20:15:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6b200d8f835a48839d2bcc136acdb293, entries=150, sequenceid=147, filesize=11.9 K 2024-12-10T20:15:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fa895558d9876220a3ffb0096daff23e in 622ms, sequenceid=147, compaction requested=true 2024-12-10T20:15:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:37,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:37,734 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:37,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:37,734 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:37,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:37,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:37,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:37,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,737 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:37,737 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:37,737 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:37,737 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:37,737 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,737 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f960a9d32e5b47998bdaf3d0b8226cdc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=129.6 K 2024-12-10T20:15:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,738 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,738 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,738 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f960a9d32e5b47998bdaf3d0b8226cdc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd] 2024-12-10T20:15:37,738 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e34e95bdcb7c4996befef1c5a9c3bf75, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5b56f9f66e39449b85cdd47c6116cf83, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7c8ec1b86dc2498f9e8a17f31ad0313d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/87fcc0d3a79f4680b6babcf25122e44b] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=44.8 K 2024-12-10T20:15:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,745 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f960a9d32e5b47998bdaf3d0b8226cdc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861732181 2024-12-10T20:15:37,745 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e34e95bdcb7c4996befef1c5a9c3bf75, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861732181 2024-12-10T20:15:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,745 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b56f9f66e39449b85cdd47c6116cf83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733861733490 2024-12-10T20:15:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,746 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8ec1b86dc2498f9e8a17f31ad0313d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733861734657 2024-12-10T20:15:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,746 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 87fcc0d3a79f4680b6babcf25122e44b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733861735936 2024-12-10T20:15:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,749 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e3f01c9814049e698e03895889d0c36, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733861733481 2024-12-10T20:15:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,749 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad031c3690d496ebdf874569684d135, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733861734657 2024-12-10T20:15:37,750 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9b3fd9e4ac4bcab478b83608bc35cd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733861735936 2024-12-10T20:15:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,773 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,774 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0636a3070a7948f7be3b53001ec54e19 is 50, key is test_row_0/B:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,789 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:37,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:15:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:37,793 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121002b0adb119be497d94f6018ec284f781_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,795 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121002b0adb119be497d94f6018ec284f781_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,796 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121002b0adb119be497d94f6018ec284f781_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741979_1155 (size=12391) 2024-12-10T20:15:37,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103c2490bb11874ff1844817ac43aa91e1_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861737135/Put/seqid=0 2024-12-10T20:15:37,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741980_1156 (size=4469) 2024-12-10T20:15:37,822 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#133 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:37,823 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/a535bc0bf4a5468e8e3bc78adf2806e0 is 175, key is test_row_0/A:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741982_1158 (size=31345) 2024-12-10T20:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741981_1157 (size=12304) 2024-12-10T20:15:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,853 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0636a3070a7948f7be3b53001ec54e19 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0636a3070a7948f7be3b53001ec54e19 2024-12-10T20:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,868 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/a535bc0bf4a5468e8e3bc78adf2806e0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a535bc0bf4a5468e8e3bc78adf2806e0 2024-12-10T20:15:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,869 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into 0636a3070a7948f7be3b53001ec54e19(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:37,869 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:37,870 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=12, startTime=1733861737734; duration=0sec 2024-12-10T20:15:37,870 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:37,870 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:37,870 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,877 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:37,877 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:37,877 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,877 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7b165492d7c64c918f172e9265d602f3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/da8b101d5ea54df39cff213fa597fcab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/0d3a6b978b614fb7bfced8dd57a7691e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6b200d8f835a48839d2bcc136acdb293] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=44.8 K 2024-12-10T20:15:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,881 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into a535bc0bf4a5468e8e3bc78adf2806e0(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:37,881 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:37,881 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=12, startTime=1733861737734; duration=0sec 2024-12-10T20:15:37,881 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:37,881 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,882 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b165492d7c64c918f172e9265d602f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733861732181 2024-12-10T20:15:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,888 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting da8b101d5ea54df39cff213fa597fcab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733861733490 2024-12-10T20:15:37,889 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3a6b978b614fb7bfced8dd57a7691e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733861734657 2024-12-10T20:15:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,890 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b200d8f835a48839d2bcc136acdb293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733861735936 2024-12-10T20:15:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,919 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,919 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/b26a9264945e4ed5bf84a05215857064 is 50, key is test_row_0/C:col10/1733861735936/Put/seqid=0 2024-12-10T20:15:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741983_1159 (size=12391) 2024-12-10T20:15:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:37,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:37,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:37,967 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/b26a9264945e4ed5bf84a05215857064 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/b26a9264945e4ed5bf84a05215857064 2024-12-10T20:15:37,989 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into b26a9264945e4ed5bf84a05215857064(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:37,989 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:37,989 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=12, startTime=1733861737735; duration=0sec 2024-12-10T20:15:37,989 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:37,989 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:38,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861738022, max timestamp=1733861738064Num uniques 859; 2024-12-10T20:15:38,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861738022, max timestamp=1733861738064Num uniques 859; 2024-12-10T20:15:38,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:38,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:38,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861738022, max timestamp=1733861738064Num uniques 859; 2024-12-10T20:15:38,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:38,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:38,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861798161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:38,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:38,257 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103c2490bb11874ff1844817ac43aa91e1_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103c2490bb11874ff1844817ac43aa91e1_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:38,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f08a25cbbab146e29a7cac3686fdd520, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:38,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f08a25cbbab146e29a7cac3686fdd520 is 175, key is test_row_0/A:col10/1733861737135/Put/seqid=0 2024-12-10T20:15:38,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861798278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:38,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:38,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:38,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741984_1160 (size=31101) 2024-12-10T20:15:38,305 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f08a25cbbab146e29a7cac3686fdd520 2024-12-10T20:15:38,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/511f15a9e3194f1a93d73c4e20aba7e3 is 50, key is test_row_0/B:col10/1733861737135/Put/seqid=0 2024-12-10T20:15:38,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741985_1161 (size=9757) 2024-12-10T20:15:38,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/511f15a9e3194f1a93d73c4e20aba7e3 2024-12-10T20:15:38,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/5de484fe043c48e4ab42b782210201d8 is 50, key is test_row_0/C:col10/1733861737135/Put/seqid=0 2024-12-10T20:15:38,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:38,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:38,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741986_1162 (size=9757) 2024-12-10T20:15:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861798488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:38,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861798797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:38,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/5de484fe043c48e4ab42b782210201d8 2024-12-10T20:15:38,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f08a25cbbab146e29a7cac3686fdd520 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520 2024-12-10T20:15:38,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:38,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:38,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520, entries=150, sequenceid=157, filesize=30.4 K 2024-12-10T20:15:38,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/511f15a9e3194f1a93d73c4e20aba7e3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/511f15a9e3194f1a93d73c4e20aba7e3 2024-12-10T20:15:38,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/511f15a9e3194f1a93d73c4e20aba7e3, entries=100, sequenceid=157, filesize=9.5 K 2024-12-10T20:15:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/5de484fe043c48e4ab42b782210201d8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5de484fe043c48e4ab42b782210201d8 2024-12-10T20:15:38,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5de484fe043c48e4ab42b782210201d8, entries=100, sequenceid=157, filesize=9.5 K 2024-12-10T20:15:38,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for fa895558d9876220a3ffb0096daff23e in 1180ms, sequenceid=157, compaction requested=false 2024-12-10T20:15:38,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:39,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T20:15:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-10T20:15:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106c0f0a2435674e82849020c04f3d581e_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861738135/Put/seqid=0 2024-12-10T20:15:39,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741987_1163 (size=12304) 2024-12-10T20:15:39,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:39,153 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106c0f0a2435674e82849020c04f3d581e_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106c0f0a2435674e82849020c04f3d581e_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:39,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/848a0b57902249719b717bf9a27071c5, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/848a0b57902249719b717bf9a27071c5 is 175, key is test_row_0/A:col10/1733861738135/Put/seqid=0 2024-12-10T20:15:39,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741988_1164 (size=31105) 2024-12-10T20:15:39,197 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/848a0b57902249719b717bf9a27071c5 2024-12-10T20:15:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/c8dcfdef15104210b77500c5ee564c1a is 50, key is test_row_0/B:col10/1733861738135/Put/seqid=0 2024-12-10T20:15:39,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741989_1165 (size=12151) 2024-12-10T20:15:39,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861799315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861799315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861799316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861799314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861799344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,353 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861799429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861799429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861799440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:39,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861799637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861799641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861799657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:39,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:39,660 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/c8dcfdef15104210b77500c5ee564c1a 2024-12-10T20:15:39,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/2da084147ed74ba4ad10402f55ffa949 is 50, key is test_row_0/C:col10/1733861738135/Put/seqid=0 2024-12-10T20:15:39,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741990_1166 (size=12151) 2024-12-10T20:15:39,725 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/2da084147ed74ba4ad10402f55ffa949 2024-12-10T20:15:39,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/848a0b57902249719b717bf9a27071c5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5 2024-12-10T20:15:39,761 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5, entries=150, sequenceid=196, filesize=30.4 K 2024-12-10T20:15:39,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/c8dcfdef15104210b77500c5ee564c1a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c8dcfdef15104210b77500c5ee564c1a 2024-12-10T20:15:39,770 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c8dcfdef15104210b77500c5ee564c1a, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T20:15:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/2da084147ed74ba4ad10402f55ffa949 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2da084147ed74ba4ad10402f55ffa949 2024-12-10T20:15:39,780 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2da084147ed74ba4ad10402f55ffa949, entries=150, sequenceid=196, filesize=11.9 K 2024-12-10T20:15:39,785 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for fa895558d9876220a3ffb0096daff23e in 715ms, sequenceid=196, compaction requested=true 2024-12-10T20:15:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-10T20:15:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-10T20:15:39,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-10T20:15:39,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4700 sec 2024-12-10T20:15:39,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4790 sec 2024-12-10T20:15:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:39,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:15:39,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:39,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:39,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108f63001f9c924515a0f154f598bf2926_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741991_1167 (size=14794) 2024-12-10T20:15:40,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:40,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861739986, max timestamp=1733861740024Num uniques 859; 2024-12-10T20:15:40,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:40,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861739986, max timestamp=1733861740024Num uniques 859; 2024-12-10T20:15:40,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:40,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861739986, max timestamp=1733861740031Num uniques 859; 2024-12-10T20:15:40,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:40,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108f63001f9c924515a0f154f598bf2926_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f63001f9c924515a0f154f598bf2926_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:40,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/a8e72042d5ff4f059f93f9f5a381599f, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:40,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/a8e72042d5ff4f059f93f9f5a381599f is 175, key is test_row_0/A:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741992_1168 (size=39749) 2024-12-10T20:15:40,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/a8e72042d5ff4f059f93f9f5a381599f 2024-12-10T20:15:40,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/817d54e5b31f437a89c1d2e47741d6b1 is 50, key is test_row_0/B:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861800093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741993_1169 (size=12151) 2024-12-10T20:15:40,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/817d54e5b31f437a89c1d2e47741d6b1 2024-12-10T20:15:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861800125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861800125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/1d307ecafb204592b271b511425a124e is 50, key is test_row_0/C:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741994_1170 (size=12151) 2024-12-10T20:15:40,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/1d307ecafb204592b271b511425a124e 2024-12-10T20:15:40,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/a8e72042d5ff4f059f93f9f5a381599f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f 2024-12-10T20:15:40,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f, entries=200, sequenceid=207, filesize=38.8 K 2024-12-10T20:15:40,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/817d54e5b31f437a89c1d2e47741d6b1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/817d54e5b31f437a89c1d2e47741d6b1 2024-12-10T20:15:40,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/817d54e5b31f437a89c1d2e47741d6b1, entries=150, sequenceid=207, filesize=11.9 K 2024-12-10T20:15:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/1d307ecafb204592b271b511425a124e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/1d307ecafb204592b271b511425a124e 2024-12-10T20:15:40,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/1d307ecafb204592b271b511425a124e, entries=150, sequenceid=207, filesize=11.9 K 2024-12-10T20:15:40,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for fa895558d9876220a3ffb0096daff23e in 285ms, sequenceid=207, compaction requested=true 2024-12-10T20:15:40,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:40,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:40,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:40,236 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:40,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:40,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:40,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:40,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:15:40,238 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:40,238 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:40,239 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:40,239 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:40,239 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a535bc0bf4a5468e8e3bc78adf2806e0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=130.2 K 2024-12-10T20:15:40,239 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:40,239 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a535bc0bf4a5468e8e3bc78adf2806e0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f] 2024-12-10T20:15:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:40,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T20:15:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:40,253 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a535bc0bf4a5468e8e3bc78adf2806e0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733861735936 2024-12-10T20:15:40,256 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f08a25cbbab146e29a7cac3686fdd520, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733861737135 2024-12-10T20:15:40,256 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:40,257 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:40,257 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:40,257 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0636a3070a7948f7be3b53001ec54e19, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/511f15a9e3194f1a93d73c4e20aba7e3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c8dcfdef15104210b77500c5ee564c1a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/817d54e5b31f437a89c1d2e47741d6b1] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=45.4 K 2024-12-10T20:15:40,257 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 848a0b57902249719b717bf9a27071c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733861738135 2024-12-10T20:15:40,259 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e72042d5ff4f059f93f9f5a381599f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733861739307 2024-12-10T20:15:40,259 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0636a3070a7948f7be3b53001ec54e19, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733861735936 2024-12-10T20:15:40,260 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 511f15a9e3194f1a93d73c4e20aba7e3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733861737135 2024-12-10T20:15:40,261 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c8dcfdef15104210b77500c5ee564c1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733861738135 2024-12-10T20:15:40,262 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 817d54e5b31f437a89c1d2e47741d6b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733861739311 2024-12-10T20:15:40,275 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:40,282 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#145 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:40,282 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/b4a2afb6e059462d94395938211863d7 is 50, key is test_row_0/B:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,285 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210065717b717644d58b7aa8d4184d99eb4_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:40,288 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210065717b717644d58b7aa8d4184d99eb4_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:40,288 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210065717b717644d58b7aa8d4184d99eb4_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:40,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fe77e024c83d40c3a3e7ec87cc157ab0_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861740123/Put/seqid=0 2024-12-10T20:15:40,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861800290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861800292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861800298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861800338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,340 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741996_1172 (size=4469) 2024-12-10T20:15:40,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741997_1173 (size=14794) 2024-12-10T20:15:40,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:40,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741995_1171 (size=12527) 2024-12-10T20:15:40,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fe77e024c83d40c3a3e7ec87cc157ab0_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fe77e024c83d40c3a3e7ec87cc157ab0_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:40,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/b88a134cd70a4ac397a102e2c1ee69e3, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:40,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/b88a134cd70a4ac397a102e2c1ee69e3 is 175, key is test_row_0/A:col10/1733861740123/Put/seqid=0 2024-12-10T20:15:40,404 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/b4a2afb6e059462d94395938211863d7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/b4a2afb6e059462d94395938211863d7 2024-12-10T20:15:40,413 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into b4a2afb6e059462d94395938211863d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:40,413 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:40,413 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=12, startTime=1733861740236; duration=0sec 2024-12-10T20:15:40,413 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:40,413 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:40,413 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:40,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861800407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,418 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:40,418 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:40,418 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:40,419 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/b26a9264945e4ed5bf84a05215857064, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5de484fe043c48e4ab42b782210201d8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2da084147ed74ba4ad10402f55ffa949, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/1d307ecafb204592b271b511425a124e] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=45.4 K 2024-12-10T20:15:40,420 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b26a9264945e4ed5bf84a05215857064, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733861735936 2024-12-10T20:15:40,420 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de484fe043c48e4ab42b782210201d8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733861737135 2024-12-10T20:15:40,421 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da084147ed74ba4ad10402f55ffa949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733861738135 2024-12-10T20:15:40,422 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d307ecafb204592b271b511425a124e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733861739311 2024-12-10T20:15:40,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861800417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861800417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,455 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:40,456 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/9e08e3caceb948a9ac1b950f0801de0a is 50, key is test_row_0/C:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741998_1174 (size=39749) 2024-12-10T20:15:40,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/b88a134cd70a4ac397a102e2c1ee69e3 2024-12-10T20:15:40,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741999_1175 (size=12527) 2024-12-10T20:15:40,504 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/9e08e3caceb948a9ac1b950f0801de0a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/9e08e3caceb948a9ac1b950f0801de0a 2024-12-10T20:15:40,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0b330595ccaf48edb2ee0e29fe94ac35 is 50, key is test_row_0/B:col10/1733861740123/Put/seqid=0 2024-12-10T20:15:40,528 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into 9e08e3caceb948a9ac1b950f0801de0a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:40,528 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:40,528 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=12, startTime=1733861740236; duration=0sec 2024-12-10T20:15:40,528 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:40,528 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:40,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742000_1176 (size=12151) 2024-12-10T20:15:40,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0b330595ccaf48edb2ee0e29fe94ac35 2024-12-10T20:15:40,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/26033eae457e43fa9a8cb2b9bec68d32 is 50, key is test_row_0/C:col10/1733861740123/Put/seqid=0 2024-12-10T20:15:40,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861800627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861800637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861800637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:40,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742001_1177 (size=12151) 2024-12-10T20:15:40,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/26033eae457e43fa9a8cb2b9bec68d32 2024-12-10T20:15:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/b88a134cd70a4ac397a102e2c1ee69e3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3 2024-12-10T20:15:40,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3, entries=200, sequenceid=244, filesize=38.8 K 2024-12-10T20:15:40,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0b330595ccaf48edb2ee0e29fe94ac35 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0b330595ccaf48edb2ee0e29fe94ac35 2024-12-10T20:15:40,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0b330595ccaf48edb2ee0e29fe94ac35, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T20:15:40,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/26033eae457e43fa9a8cb2b9bec68d32 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/26033eae457e43fa9a8cb2b9bec68d32 2024-12-10T20:15:40,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/26033eae457e43fa9a8cb2b9bec68d32, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T20:15:40,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fa895558d9876220a3ffb0096daff23e in 510ms, sequenceid=244, compaction requested=false 2024-12-10T20:15:40,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:40,772 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#144 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:40,773 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/0cbb283536b84b10ae71ab9dd7c42199 is 175, key is test_row_0/A:col10/1733861739311/Put/seqid=0 2024-12-10T20:15:40,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742002_1178 (size=31481) 2024-12-10T20:15:40,845 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/0cbb283536b84b10ae71ab9dd7c42199 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0cbb283536b84b10ae71ab9dd7c42199 2024-12-10T20:15:40,873 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into 0cbb283536b84b10ae71ab9dd7c42199(size=30.7 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:40,873 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:40,873 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=12, startTime=1733861740235; duration=0sec 2024-12-10T20:15:40,873 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:40,873 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:40,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:15:40,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:40,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:40,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:40,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:40,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:40,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:41,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210451522195d04476fb295679b072be042_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861740967, max timestamp=1733861741047Num uniques 859; 2024-12-10T20:15:41,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:41,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861740967, max timestamp=1733861741047Num uniques 859; 2024-12-10T20:15:41,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:41,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861740967, max timestamp=1733861741047Num uniques 859; 2024-12-10T20:15:41,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:41,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742003_1179 (size=14894) 2024-12-10T20:15:41,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:41,083 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210451522195d04476fb295679b072be042_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210451522195d04476fb295679b072be042_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:41,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/ce1a46d30f6347ffafa4c03acd365194, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:41,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/ce1a46d30f6347ffafa4c03acd365194 is 175, key is test_row_0/A:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861801112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742004_1180 (size=39849) 2024-12-10T20:15:41,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861801120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861801120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/ce1a46d30f6347ffafa4c03acd365194 2024-12-10T20:15:41,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/94d984e53070496187d238072c574f8e is 50, key is test_row_0/B:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742005_1181 (size=12251) 2024-12-10T20:15:41,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/94d984e53070496187d238072c574f8e 2024-12-10T20:15:41,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861801234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/ff9d623bd44d47b2bb4871f382ee4295 is 50, key is test_row_0/C:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861801245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861801245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742006_1182 (size=12251) 2024-12-10T20:15:41,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/ff9d623bd44d47b2bb4871f382ee4295 2024-12-10T20:15:41,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/ce1a46d30f6347ffafa4c03acd365194 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194 2024-12-10T20:15:41,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194, entries=200, sequenceid=258, filesize=38.9 K 2024-12-10T20:15:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/94d984e53070496187d238072c574f8e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/94d984e53070496187d238072c574f8e 2024-12-10T20:15:41,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/94d984e53070496187d238072c574f8e, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T20:15:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/ff9d623bd44d47b2bb4871f382ee4295 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/ff9d623bd44d47b2bb4871f382ee4295 2024-12-10T20:15:41,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/ff9d623bd44d47b2bb4871f382ee4295, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T20:15:41,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fa895558d9876220a3ffb0096daff23e in 406ms, sequenceid=258, compaction requested=true 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T20:15:41,369 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:41,373 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:41,373 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111079 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:41,374 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:41,374 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,374 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0cbb283536b84b10ae71ab9dd7c42199, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=108.5 K 2024-12-10T20:15:41,374 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,374 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0cbb283536b84b10ae71ab9dd7c42199, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194] 2024-12-10T20:15:41,375 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbb283536b84b10ae71ab9dd7c42199, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733861739311 2024-12-10T20:15:41,380 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b88a134cd70a4ac397a102e2c1ee69e3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861740093 2024-12-10T20:15:41,381 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:41,381 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:41,381 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,382 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/9e08e3caceb948a9ac1b950f0801de0a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/26033eae457e43fa9a8cb2b9bec68d32, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/ff9d623bd44d47b2bb4871f382ee4295] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=36.1 K 2024-12-10T20:15:41,383 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1a46d30f6347ffafa4c03acd365194, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733861740265 2024-12-10T20:15:41,393 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e08e3caceb948a9ac1b950f0801de0a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733861739311 2024-12-10T20:15:41,393 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26033eae457e43fa9a8cb2b9bec68d32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861740093 2024-12-10T20:15:41,397 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff9d623bd44d47b2bb4871f382ee4295, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733861740265 2024-12-10T20:15:41,424 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:41,424 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/e009a08c619e45fdb57cf3722ee1f768 is 50, key is test_row_0/C:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,453 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:41,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T20:15:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:41,468 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121064cf8826bb6549829e52ad25f9c3b6cd_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:41,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742007_1183 (size=12729) 2024-12-10T20:15:41,478 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121064cf8826bb6549829e52ad25f9c3b6cd_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:41,478 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121064cf8826bb6549829e52ad25f9c3b6cd_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T20:15:41,493 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:41,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-10T20:15:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:41,502 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:41,503 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:41,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:41,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861801501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861801503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861801505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121029378202cb714249bed0897834813cac_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861741117/Put/seqid=0 2024-12-10T20:15:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742008_1184 (size=4469) 2024-12-10T20:15:41,565 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#154 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:41,566 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/92ca4d450ba74955b5863840208d3403 is 175, key is test_row_0/A:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742009_1185 (size=12454) 2024-12-10T20:15:41,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:41,605 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121029378202cb714249bed0897834813cac_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121029378202cb714249bed0897834813cac_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:41,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/466987f21fd34b93b8c88ebb48c45a15, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:41,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/466987f21fd34b93b8c88ebb48c45a15 is 175, key is test_row_0/A:col10/1733861741117/Put/seqid=0 2024-12-10T20:15:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:41,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861801621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861801629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861801639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742010_1186 (size=31683) 2024-12-10T20:15:41,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742011_1187 (size=31255) 2024-12-10T20:15:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:41,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:41,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:41,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861801848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861801850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861801859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:41,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:41,889 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/e009a08c619e45fdb57cf3722ee1f768 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e009a08c619e45fdb57cf3722ee1f768 2024-12-10T20:15:41,900 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into e009a08c619e45fdb57cf3722ee1f768(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:41,900 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:41,900 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=13, startTime=1733861741369; duration=0sec 2024-12-10T20:15:41,900 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:41,900 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:41,900 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:41,905 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:41,905 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:41,905 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,905 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/b4a2afb6e059462d94395938211863d7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0b330595ccaf48edb2ee0e29fe94ac35, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/94d984e53070496187d238072c574f8e] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=36.1 K 2024-12-10T20:15:41,906 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a2afb6e059462d94395938211863d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733861739311 2024-12-10T20:15:41,907 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b330595ccaf48edb2ee0e29fe94ac35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861740093 2024-12-10T20:15:41,908 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d984e53070496187d238072c574f8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733861740265 2024-12-10T20:15:41,931 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:41,932 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/cf2d2916845e4acea8d04905a26c2c3e is 50, key is test_row_0/B:col10/1733861740963/Put/seqid=0 2024-12-10T20:15:41,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:41,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:41,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742012_1188 (size=12729) 2024-12-10T20:15:42,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/466987f21fd34b93b8c88ebb48c45a15 2024-12-10T20:15:42,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/5909b2e1238f472d91744d90c92a606b is 50, key is test_row_0/B:col10/1733861741117/Put/seqid=0 2024-12-10T20:15:42,104 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/92ca4d450ba74955b5863840208d3403 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/92ca4d450ba74955b5863840208d3403 2024-12-10T20:15:42,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742013_1189 (size=12301) 2024-12-10T20:15:42,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:42,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,147 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into 92ca4d450ba74955b5863840208d3403(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:42,147 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:42,147 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=13, startTime=1733861741369; duration=0sec 2024-12-10T20:15:42,147 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:42,147 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:42,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861802167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861802187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861802187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:42,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:42,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861802357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,362 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,421 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/cf2d2916845e4acea8d04905a26c2c3e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf2d2916845e4acea8d04905a26c2c3e 2024-12-10T20:15:42,431 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into cf2d2916845e4acea8d04905a26c2c3e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:42,431 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:42,431 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=13, startTime=1733861741369; duration=0sec 2024-12-10T20:15:42,431 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:42,431 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:42,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/5909b2e1238f472d91744d90c92a606b 2024-12-10T20:15:42,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/f699b798d23a457aa3c999379af67b0f is 50, key is test_row_0/C:col10/1733861741117/Put/seqid=0 2024-12-10T20:15:42,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742014_1190 (size=12301) 2024-12-10T20:15:42,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861802687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861802697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861802697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:42,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:42,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:42,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:42,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:42,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:43,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/f699b798d23a457aa3c999379af67b0f 2024-12-10T20:15:43,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/466987f21fd34b93b8c88ebb48c45a15 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15 2024-12-10T20:15:43,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15, entries=150, sequenceid=294, filesize=30.5 K 2024-12-10T20:15:43,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/5909b2e1238f472d91744d90c92a606b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5909b2e1238f472d91744d90c92a606b 2024-12-10T20:15:43,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5909b2e1238f472d91744d90c92a606b, entries=150, sequenceid=294, filesize=12.0 K 2024-12-10T20:15:43,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/f699b798d23a457aa3c999379af67b0f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f699b798d23a457aa3c999379af67b0f 2024-12-10T20:15:43,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f699b798d23a457aa3c999379af67b0f, entries=150, sequenceid=294, filesize=12.0 K 2024-12-10T20:15:43,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for fa895558d9876220a3ffb0096daff23e in 1575ms, sequenceid=294, compaction requested=false 2024-12-10T20:15:43,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:43,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T20:15:43,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:43,121 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:15:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:43,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:43,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210acd77cf5e26440288ea17823447acbee_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861741491/Put/seqid=0 2024-12-10T20:15:43,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742015_1191 (size=12454) 2024-12-10T20:15:43,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:43,619 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210acd77cf5e26440288ea17823447acbee_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210acd77cf5e26440288ea17823447acbee_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:43,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/65eb720f50694fcdbd7d5bde7cbe8260, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/65eb720f50694fcdbd7d5bde7cbe8260 is 175, key is test_row_0/A:col10/1733861741491/Put/seqid=0 2024-12-10T20:15:43,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742016_1192 (size=31255) 2024-12-10T20:15:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:43,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861743744, max timestamp=1733861743806Num uniques 859; 2024-12-10T20:15:43,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:43,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861743744, max timestamp=1733861743806Num uniques 859; 2024-12-10T20:15:43,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:43,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861743744, max timestamp=1733861743806Num uniques 859; 2024-12-10T20:15:43,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861803892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861803899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861803911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861804033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861804041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861804041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,065 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/65eb720f50694fcdbd7d5bde7cbe8260 2024-12-10T20:15:44,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/676ea811968a46bbbcd88aef292f9201 is 50, key is test_row_0/B:col10/1733861741491/Put/seqid=0 2024-12-10T20:15:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742017_1193 (size=12301) 2024-12-10T20:15:44,150 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/676ea811968a46bbbcd88aef292f9201 2024-12-10T20:15:44,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/aece297414964b5abdc4437a7eb51b02 is 50, key is test_row_0/C:col10/1733861741491/Put/seqid=0 2024-12-10T20:15:44,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742018_1194 (size=12301) 2024-12-10T20:15:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861804261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861804265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861804265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861804577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861804577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861804577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:44,646 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/aece297414964b5abdc4437a7eb51b02 2024-12-10T20:15:44,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/65eb720f50694fcdbd7d5bde7cbe8260 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260 2024-12-10T20:15:44,667 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260, entries=150, sequenceid=308, filesize=30.5 K 2024-12-10T20:15:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/676ea811968a46bbbcd88aef292f9201 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/676ea811968a46bbbcd88aef292f9201 2024-12-10T20:15:44,688 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/676ea811968a46bbbcd88aef292f9201, entries=150, sequenceid=308, filesize=12.0 K 2024-12-10T20:15:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/aece297414964b5abdc4437a7eb51b02 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/aece297414964b5abdc4437a7eb51b02 2024-12-10T20:15:44,729 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/aece297414964b5abdc4437a7eb51b02, entries=150, sequenceid=308, filesize=12.0 K 2024-12-10T20:15:44,737 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fa895558d9876220a3ffb0096daff23e in 1616ms, sequenceid=308, compaction requested=true 2024-12-10T20:15:44,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:44,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:44,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-10T20:15:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-10T20:15:44,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-10T20:15:44,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2360 sec 2024-12-10T20:15:44,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.2530 sec 2024-12-10T20:15:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:45,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-10T20:15:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T20:15:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T20:15:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:45,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T20:15:45,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121002b6b4806723466da9192dc821667ae9_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:45,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861805128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861805131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861805138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742019_1195 (size=17534) 2024-12-10T20:15:45,186 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:45,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121002b6b4806723466da9192dc821667ae9_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121002b6b4806723466da9192dc821667ae9_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:45,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/aa4e7134f5b84b97bccc0d7c77d3000e, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:45,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/aa4e7134f5b84b97bccc0d7c77d3000e is 175, key is test_row_0/A:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:45,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861805249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861805249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742020_1196 (size=48639) 2024-12-10T20:15:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861805249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,269 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/aa4e7134f5b84b97bccc0d7c77d3000e 2024-12-10T20:15:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/2f5efaa7f1b241ac96e9e5258af93bb9 is 50, key is test_row_0/B:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:45,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742021_1197 (size=12301) 2024-12-10T20:15:45,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861805469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861805469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861805476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T20:15:45,700 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:45,700 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-10T20:15:45,703 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:45,704 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:45,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T20:15:45,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/2f5efaa7f1b241ac96e9e5258af93bb9 2024-12-10T20:15:45,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/5c1c6963f0cc41a1888db691afaeda85 is 50, key is test_row_0/C:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:45,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861805777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861805789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861805789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T20:15:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742022_1198 (size=12301) 2024-12-10T20:15:45,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/5c1c6963f0cc41a1888db691afaeda85 2024-12-10T20:15:45,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/aa4e7134f5b84b97bccc0d7c77d3000e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e 2024-12-10T20:15:45,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T20:15:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:45,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:45,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e, entries=250, sequenceid=346, filesize=47.5 K 2024-12-10T20:15:45,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/2f5efaa7f1b241ac96e9e5258af93bb9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/2f5efaa7f1b241ac96e9e5258af93bb9 2024-12-10T20:15:45,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/2f5efaa7f1b241ac96e9e5258af93bb9, entries=150, sequenceid=346, filesize=12.0 K 2024-12-10T20:15:45,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/5c1c6963f0cc41a1888db691afaeda85 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5c1c6963f0cc41a1888db691afaeda85 2024-12-10T20:15:45,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5c1c6963f0cc41a1888db691afaeda85, entries=150, sequenceid=346, filesize=12.0 K 2024-12-10T20:15:45,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for fa895558d9876220a3ffb0096daff23e in 826ms, sequenceid=346, compaction requested=true 2024-12-10T20:15:45,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:45,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:45,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:45,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:45,921 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:45,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:15:45,921 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:45,923 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142832 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:45,923 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:45,923 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:45,923 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/92ca4d450ba74955b5863840208d3403, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=139.5 K 2024-12-10T20:15:45,923 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:45,923 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/92ca4d450ba74955b5863840208d3403, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e] 2024-12-10T20:15:45,924 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:45,924 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:45,925 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:45,925 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf2d2916845e4acea8d04905a26c2c3e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5909b2e1238f472d91744d90c92a606b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/676ea811968a46bbbcd88aef292f9201, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/2f5efaa7f1b241ac96e9e5258af93bb9] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=48.5 K 2024-12-10T20:15:45,925 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ca4d450ba74955b5863840208d3403, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733861740265 2024-12-10T20:15:45,926 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2d2916845e4acea8d04905a26c2c3e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733861740265 2024-12-10T20:15:45,926 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 466987f21fd34b93b8c88ebb48c45a15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733861741117 2024-12-10T20:15:45,927 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5909b2e1238f472d91744d90c92a606b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733861741117 2024-12-10T20:15:45,927 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 65eb720f50694fcdbd7d5bde7cbe8260, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733861741474 2024-12-10T20:15:45,927 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676ea811968a46bbbcd88aef292f9201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733861741474 2024-12-10T20:15:45,928 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4e7134f5b84b97bccc0d7c77d3000e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733861743868 2024-12-10T20:15:45,929 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5efaa7f1b241ac96e9e5258af93bb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733861743889 2024-12-10T20:15:45,970 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:45,985 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:45,985 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/15455c785a4a41629cf2e30d5fb12f4e is 50, key is test_row_0/B:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:45,996 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210201ab59f0ed7424780ada46daf46099f_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:45,999 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210201ab59f0ed7424780ada46daf46099f_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:45,999 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210201ab59f0ed7424780ada46daf46099f_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:46,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742024_1200 (size=4469) 2024-12-10T20:15:46,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742023_1199 (size=12915) 2024-12-10T20:15:46,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:46,029 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T20:15:46,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105b6e8e32f49646c5a529204a864fc054_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861745125/Put/seqid=0 2024-12-10T20:15:46,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742025_1201 (size=12454) 2024-12-10T20:15:46,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:46,101 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105b6e8e32f49646c5a529204a864fc054_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105b6e8e32f49646c5a529204a864fc054_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f16748b64a0748209eaf9d7964a287df, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:46,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f16748b64a0748209eaf9d7964a287df is 175, key is test_row_0/A:col10/1733861745125/Put/seqid=0 2024-12-10T20:15:46,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742026_1202 (size=31255) 2024-12-10T20:15:46,137 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f16748b64a0748209eaf9d7964a287df 2024-12-10T20:15:46,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/7f64e8492978454591eef72e97dccd1d is 50, key is test_row_0/B:col10/1733861745125/Put/seqid=0 2024-12-10T20:15:46,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742027_1203 (size=12301) 2024-12-10T20:15:46,251 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/7f64e8492978454591eef72e97dccd1d 2024-12-10T20:15:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/4df18eb858a84aeba802527d0143fac8 is 50, key is test_row_0/C:col10/1733861745125/Put/seqid=0 2024-12-10T20:15:46,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:46,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742028_1204 (size=12301) 2024-12-10T20:15:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T20:15:46,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861746341, max timestamp=1733861746374Num uniques 859; 2024-12-10T20:15:46,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:15:46,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861746341, max timestamp=1733861746374Num uniques 859; 2024-12-10T20:15:46,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:15:46,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861746341, max timestamp=1733861746374Num uniques 859; 2024-12-10T20:15:46,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:15:46,419 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#165 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:46,420 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/d381edd21b034ff7ad0c6ca4739d9c4c is 175, key is test_row_0/A:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:46,433 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/15455c785a4a41629cf2e30d5fb12f4e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/15455c785a4a41629cf2e30d5fb12f4e 2024-12-10T20:15:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742029_1205 (size=31869) 2024-12-10T20:15:46,469 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into 15455c785a4a41629cf2e30d5fb12f4e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:46,469 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:46,469 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=12, startTime=1733861745920; duration=0sec 2024-12-10T20:15:46,469 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:46,469 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:46,469 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:46,471 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:46,471 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:46,471 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:46,471 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e009a08c619e45fdb57cf3722ee1f768, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f699b798d23a457aa3c999379af67b0f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/aece297414964b5abdc4437a7eb51b02, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5c1c6963f0cc41a1888db691afaeda85] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=48.5 K 2024-12-10T20:15:46,472 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e009a08c619e45fdb57cf3722ee1f768, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733861740265 2024-12-10T20:15:46,472 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f699b798d23a457aa3c999379af67b0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733861741117 2024-12-10T20:15:46,473 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting aece297414964b5abdc4437a7eb51b02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733861741474 2024-12-10T20:15:46,473 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1c6963f0cc41a1888db691afaeda85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733861743889 2024-12-10T20:15:46,475 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/d381edd21b034ff7ad0c6ca4739d9c4c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d381edd21b034ff7ad0c6ca4739d9c4c 2024-12-10T20:15:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861806470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,489 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into d381edd21b034ff7ad0c6ca4739d9c4c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:46,489 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:46,489 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=12, startTime=1733861745920; duration=0sec 2024-12-10T20:15:46,489 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:46,489 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:46,495 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:46,496 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/6a2f500a7e44456796bd16a5aa06f37e is 50, key is test_row_0/C:col10/1733861743889/Put/seqid=0 2024-12-10T20:15:46,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861806477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861806481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861806483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742030_1206 (size=12915) 2024-12-10T20:15:46,590 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/6a2f500a7e44456796bd16a5aa06f37e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6a2f500a7e44456796bd16a5aa06f37e 2024-12-10T20:15:46,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861806587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,605 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into 6a2f500a7e44456796bd16a5aa06f37e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:46,605 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:46,605 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=12, startTime=1733861745921; duration=0sec 2024-12-10T20:15:46,605 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:46,605 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861806607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861806617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861806617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,732 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/4df18eb858a84aeba802527d0143fac8 2024-12-10T20:15:46,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f16748b64a0748209eaf9d7964a287df as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df 2024-12-10T20:15:46,801 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df, entries=150, sequenceid=355, filesize=30.5 K 2024-12-10T20:15:46,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861806797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/7f64e8492978454591eef72e97dccd1d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7f64e8492978454591eef72e97dccd1d 2024-12-10T20:15:46,822 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7f64e8492978454591eef72e97dccd1d, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T20:15:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/4df18eb858a84aeba802527d0143fac8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/4df18eb858a84aeba802527d0143fac8 2024-12-10T20:15:46,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861806837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861806838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861806838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:46,847 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/4df18eb858a84aeba802527d0143fac8, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T20:15:46,860 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=143.13 KB/146560 for fa895558d9876220a3ffb0096daff23e in 830ms, sequenceid=355, compaction requested=false 2024-12-10T20:15:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-10T20:15:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-10T20:15:46,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-10T20:15:46,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-12-10T20:15:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T20:15:46,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1660 sec 2024-12-10T20:15:47,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861746424, max timestamp=1733861747119Num uniques 1118; 2024-12-10T20:15:47,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-10T20:15:47,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861746424, max timestamp=1733861747119Num uniques 1118; 2024-12-10T20:15:47,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-10T20:15:47,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861746424, max timestamp=1733861747119Num uniques 1018; 2024-12-10T20:15:47,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T20:15:47,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-10T20:15:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:47,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c64c6d23f23542eaa48e4dd2ac1bf419_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:47,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742031_1207 (size=20074) 2024-12-10T20:15:47,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:47,225 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c64c6d23f23542eaa48e4dd2ac1bf419_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c64c6d23f23542eaa48e4dd2ac1bf419_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:47,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/811e7dd1772e43d3ab315837f7e285a9, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:47,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/811e7dd1772e43d3ab315837f7e285a9 is 175, key is test_row_0/A:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:47,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861747205, max timestamp=1733861747239Num uniques 859; 2024-12-10T20:15:47,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:47,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861747205, max timestamp=1733861747239Num uniques 859; 2024-12-10T20:15:47,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861747205, max timestamp=1733861747239Num uniques 859; 2024-12-10T20:15:47,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:47,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:15:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742032_1208 (size=57333) 2024-12-10T20:15:47,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/811e7dd1772e43d3ab315837f7e285a9 2024-12-10T20:15:47,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861807295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861807298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861807313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861807316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6847c0ac1f56463e8121cb9156aef1af is 50, key is test_row_0/B:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:47,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742033_1209 (size=12301) 2024-12-10T20:15:47,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6847c0ac1f56463e8121cb9156aef1af 2024-12-10T20:15:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861807428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7f706c4329954d0c81551dc9379c6cab is 50, key is test_row_0/C:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:47,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861807428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861807428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861807429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742034_1210 (size=12301) 2024-12-10T20:15:47,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861807648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861807649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861807649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861807649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T20:15:47,881 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:47,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-10T20:15:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T20:15:47,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:47,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:47,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:47,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7f706c4329954d0c81551dc9379c6cab 2024-12-10T20:15:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/811e7dd1772e43d3ab315837f7e285a9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9 2024-12-10T20:15:47,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9, entries=300, sequenceid=400, filesize=56.0 K 2024-12-10T20:15:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6847c0ac1f56463e8121cb9156aef1af as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6847c0ac1f56463e8121cb9156aef1af 2024-12-10T20:15:47,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6847c0ac1f56463e8121cb9156aef1af, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T20:15:47,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7f706c4329954d0c81551dc9379c6cab as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7f706c4329954d0c81551dc9379c6cab 2024-12-10T20:15:47,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7f706c4329954d0c81551dc9379c6cab, entries=150, sequenceid=400, filesize=12.0 K 2024-12-10T20:15:47,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for fa895558d9876220a3ffb0096daff23e in 804ms, sequenceid=400, compaction requested=true 2024-12-10T20:15:47,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:47,960 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:47,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:47,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:47,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:47,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:47,960 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:47,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:47,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:47,961 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120457 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:47,961 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:47,961 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:47,961 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d381edd21b034ff7ad0c6ca4739d9c4c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=117.6 K 2024-12-10T20:15:47,961 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:47,961 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d381edd21b034ff7ad0c6ca4739d9c4c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9] 2024-12-10T20:15:47,961 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:47,961 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:47,962 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:47,962 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/15455c785a4a41629cf2e30d5fb12f4e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7f64e8492978454591eef72e97dccd1d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6847c0ac1f56463e8121cb9156aef1af] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=36.6 K 2024-12-10T20:15:47,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting d381edd21b034ff7ad0c6ca4739d9c4c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733861743889 2024-12-10T20:15:47,962 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 15455c785a4a41629cf2e30d5fb12f4e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733861743889 2024-12-10T20:15:47,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16748b64a0748209eaf9d7964a287df, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733861745107 2024-12-10T20:15:47,962 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f64e8492978454591eef72e97dccd1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733861745107 2024-12-10T20:15:47,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 811e7dd1772e43d3ab315837f7e285a9, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733861746476 2024-12-10T20:15:47,963 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6847c0ac1f56463e8121cb9156aef1af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733861746476 2024-12-10T20:15:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:47,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-10T20:15:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:47,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:47,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:47,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:47,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:47,974 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:47,987 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:47,988 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6a0a708e5c0249339980b26dcfa57310 is 50, key is test_row_0/B:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T20:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861807985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861807985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861807986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:47,997 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210fe9ca6896bf64b91a8366b33a55332dd_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:47,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210fe9ca6896bf64b91a8366b33a55332dd_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:48,000 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fe9ca6896bf64b91a8366b33a55332dd_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861807993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121060411d901b614c7fb45cf0b412c6b4cc_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861747313/Put/seqid=0 2024-12-10T20:15:48,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T20:15:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742035_1211 (size=13017) 2024-12-10T20:15:48,064 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6a0a708e5c0249339980b26dcfa57310 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6a0a708e5c0249339980b26dcfa57310 2024-12-10T20:15:48,071 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into 6a0a708e5c0249339980b26dcfa57310(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:48,071 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:48,071 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=13, startTime=1733861747960; duration=0sec 2024-12-10T20:15:48,072 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:48,072 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:48,072 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:15:48,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:15:48,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:48,073 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,073 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6a2f500a7e44456796bd16a5aa06f37e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/4df18eb858a84aeba802527d0143fac8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7f706c4329954d0c81551dc9379c6cab] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=36.6 K 2024-12-10T20:15:48,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2f500a7e44456796bd16a5aa06f37e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733861743889 2024-12-10T20:15:48,074 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4df18eb858a84aeba802527d0143fac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733861745107 2024-12-10T20:15:48,074 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f706c4329954d0c81551dc9379c6cab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733861746476 2024-12-10T20:15:48,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742036_1212 (size=4469) 2024-12-10T20:15:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861808101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861808107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861808107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861808107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742037_1213 (size=14994) 2024-12-10T20:15:48,117 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,127 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:48,128 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/3101da4a4aa943c08911f00a0fd5800f is 50, key is test_row_0/C:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:48,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121060411d901b614c7fb45cf0b412c6b4cc_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121060411d901b614c7fb45cf0b412c6b4cc_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:48,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/2c9a0eb87d6e43c7b6e452d6a8017dc5, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:48,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/2c9a0eb87d6e43c7b6e452d6a8017dc5 is 175, key is test_row_0/A:col10/1733861747313/Put/seqid=0 2024-12-10T20:15:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742038_1214 (size=13017) 2024-12-10T20:15:48,180 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/3101da4a4aa943c08911f00a0fd5800f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/3101da4a4aa943c08911f00a0fd5800f 2024-12-10T20:15:48,191 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into 3101da4a4aa943c08911f00a0fd5800f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:48,191 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:48,191 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=13, startTime=1733861747960; duration=0sec 2024-12-10T20:15:48,192 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:48,192 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742039_1215 (size=39949) 2024-12-10T20:15:48,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/2c9a0eb87d6e43c7b6e452d6a8017dc5 2024-12-10T20:15:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T20:15:48,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T20:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/e466074406c44ce38ba13adac4f22e4b is 50, key is test_row_0/B:col10/1733861747313/Put/seqid=0 2024-12-10T20:15:48,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742040_1216 (size=12301) 2024-12-10T20:15:48,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/e466074406c44ce38ba13adac4f22e4b 2024-12-10T20:15:48,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/8d9ae587e5de43288d0783c1a1b2e3db is 50, key is test_row_0/C:col10/1733861747313/Put/seqid=0 2024-12-10T20:15:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861808321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861808321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861808321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861808321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742041_1217 (size=12301) 2024-12-10T20:15:48,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/8d9ae587e5de43288d0783c1a1b2e3db 2024-12-10T20:15:48,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T20:15:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:48,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/2c9a0eb87d6e43c7b6e452d6a8017dc5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5 2024-12-10T20:15:48,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5, entries=200, sequenceid=437, filesize=39.0 K 2024-12-10T20:15:48,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/e466074406c44ce38ba13adac4f22e4b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e466074406c44ce38ba13adac4f22e4b 2024-12-10T20:15:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e466074406c44ce38ba13adac4f22e4b, entries=150, sequenceid=437, filesize=12.0 K 2024-12-10T20:15:48,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/8d9ae587e5de43288d0783c1a1b2e3db as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/8d9ae587e5de43288d0783c1a1b2e3db 2024-12-10T20:15:48,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/8d9ae587e5de43288d0783c1a1b2e3db, entries=150, sequenceid=437, filesize=12.0 K 2024-12-10T20:15:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fa895558d9876220a3ffb0096daff23e in 441ms, sequenceid=437, compaction requested=false 2024-12-10T20:15:48,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,486 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#174 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,487 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f626054b47a946d68cdeca0eb1c72570 is 175, key is test_row_0/A:col10/1733861746476/Put/seqid=0 2024-12-10T20:15:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T20:15:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742042_1218 (size=31971) 2024-12-10T20:15:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T20:15:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,525 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T20:15:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,530 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f626054b47a946d68cdeca0eb1c72570 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f626054b47a946d68cdeca0eb1c72570 2024-12-10T20:15:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,545 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into f626054b47a946d68cdeca0eb1c72570(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:48,545 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:48,545 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=13, startTime=1733861747959; duration=0sec 2024-12-10T20:15:48,546 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:48,546 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a3cc322581e74ac0a30b8cb5c2873ca7_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_1/A:col10/1733861747983/Put/seqid=0 2024-12-10T20:15:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742043_1219 (size=9914) 2024-12-10T20:15:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,598 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a3cc322581e74ac0a30b8cb5c2873ca7_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a3cc322581e74ac0a30b8cb5c2873ca7_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/93b435bfd277484583fb47d10ace838e, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/93b435bfd277484583fb47d10ace838e is 175, key is test_row_1/A:col10/1733861747983/Put/seqid=0 2024-12-10T20:15:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742044_1220 (size=22561) 2024-12-10T20:15:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:48,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733861748631, max timestamp=1733861748711Num uniques 859; 2024-12-10T20:15:48,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T20:15:48,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733861748631, max timestamp=1733861748711Num uniques 859; 2024-12-10T20:15:48,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T20:15:48,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733861748631, max timestamp=1733861748711Num uniques 859; 2024-12-10T20:15:48,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-10T20:15:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861808727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861808729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861808730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861808732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861808841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861808848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861808848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861808848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T20:15:49,045 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/93b435bfd277484583fb47d10ace838e 2024-12-10T20:15:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861809058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861809061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861809067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861809061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/c7840d5426e94464b519f0ad0298aaa8 is 50, key is test_row_1/B:col10/1733861747983/Put/seqid=0 2024-12-10T20:15:49,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742045_1221 (size=9857) 2024-12-10T20:15:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861809366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861809377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861809377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861809387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57340 deadline: 1733861809417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,418 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in fa895558d9876220a3ffb0096daff23e of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:15:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,520 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/c7840d5426e94464b519f0ad0298aaa8 2024-12-10T20:15:49,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c is 50, key is test_row_1/C:col10/1733861747983/Put/seqid=0 2024-12-10T20:15:49,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742046_1222 (size=9857) 2024-12-10T20:15:49,572 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c 2024-12-10T20:15:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/93b435bfd277484583fb47d10ace838e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e 2024-12-10T20:15:49,606 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e, entries=100, sequenceid=448, filesize=22.0 K 2024-12-10T20:15:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/c7840d5426e94464b519f0ad0298aaa8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c7840d5426e94464b519f0ad0298aaa8 2024-12-10T20:15:49,612 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c7840d5426e94464b519f0ad0298aaa8, entries=100, sequenceid=448, filesize=9.6 K 2024-12-10T20:15:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c 2024-12-10T20:15:49,618 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c, entries=100, sequenceid=448, filesize=9.6 K 2024-12-10T20:15:49,623 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=161.02 KB/164880 for fa895558d9876220a3ffb0096daff23e in 1098ms, sequenceid=448, compaction requested=true 2024-12-10T20:15:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-10T20:15:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-10T20:15:49,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-10T20:15:49,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7480 sec 2024-12-10T20:15:49,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7630 sec 2024-12-10T20:15:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:49,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=167.72 KB heapSize=418.97 KB 2024-12-10T20:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:15:49,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101b35ac6f7a8143518b2455bb043a3524_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861809906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861809908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861809908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861809911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:49,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742047_1223 (size=14994) 2024-12-10T20:15:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861810018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861810018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T20:15:50,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:50,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:15:50,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861810020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-10T20:15:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:50,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:15:50,031 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:15:50,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:15:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861810034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:50,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:50,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:50,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861810227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861810227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861810238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861810247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:50,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:50,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,381 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101b35ac6f7a8143518b2455bb043a3524_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101b35ac6f7a8143518b2455bb043a3524_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:50,397 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/13cddb53b4194b2abb4983efe16c3789, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:50,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/13cddb53b4194b2abb4983efe16c3789 is 175, key is test_row_0/A:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:50,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742048_1224 (size=39949) 2024-12-10T20:15:50,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861810537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861810539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861810549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:50,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861810557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:50,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/13cddb53b4194b2abb4983efe16c3789 2024-12-10T20:15:50,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/1299d257efcf4f9593491ee754ca1d9d is 50, key is test_row_0/B:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:50,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:50,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:50,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742049_1225 (size=12301) 2024-12-10T20:15:51,005 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-12-10T20:15:51,005 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:51,005 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,005 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:51,008 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-12-10T20:15:51,008 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:51,008 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,009 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:51,012 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-12-10T20:15:51,012 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:51,012 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,012 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:51,014 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-12-10T20:15:51,014 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:51,014 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,014 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:51,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:51,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:51,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57402 deadline: 1733861811047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57412 deadline: 1733861811048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57370 deadline: 1733861811057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:15:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57386 deadline: 1733861811077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:15:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:15:51,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:51,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/1299d257efcf4f9593491ee754ca1d9d 2024-12-10T20:15:51,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7139838451c94362ac49a659bbf26a42 is 50, key is test_row_0/C:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:51,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:15:51,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742050_1226 (size=12301) 2024-12-10T20:15:51,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7139838451c94362ac49a659bbf26a42 2024-12-10T20:15:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/13cddb53b4194b2abb4983efe16c3789 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789 2024-12-10T20:15:51,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789, entries=200, sequenceid=477, filesize=39.0 K 2024-12-10T20:15:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/1299d257efcf4f9593491ee754ca1d9d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/1299d257efcf4f9593491ee754ca1d9d 2024-12-10T20:15:51,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/1299d257efcf4f9593491ee754ca1d9d, entries=150, sequenceid=477, filesize=12.0 K 2024-12-10T20:15:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/7139838451c94362ac49a659bbf26a42 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7139838451c94362ac49a659bbf26a42 2024-12-10T20:15:51,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7139838451c94362ac49a659bbf26a42, entries=150, sequenceid=477, filesize=12.0 K 2024-12-10T20:15:51,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~418.92 KB/428976, currentSize=33.54 KB/34350 for fa895558d9876220a3ffb0096daff23e in 1531ms, sequenceid=477, compaction requested=true 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:15:51,414 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa895558d9876220a3ffb0096daff23e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:15:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:15:51,417 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:51,417 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:51,418 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/A is initiating minor compaction (all files) 2024-12-10T20:15:51,418 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/A in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,418 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f626054b47a946d68cdeca0eb1c72570, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=131.3 K 2024-12-10T20:15:51,418 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,418 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f626054b47a946d68cdeca0eb1c72570, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789] 2024-12-10T20:15:51,418 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f626054b47a946d68cdeca0eb1c72570, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733861746476 2024-12-10T20:15:51,419 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9a0eb87d6e43c7b6e452d6a8017dc5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733861747289 2024-12-10T20:15:51,419 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:51,419 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/B is initiating minor compaction (all files) 2024-12-10T20:15:51,419 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b435bfd277484583fb47d10ace838e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733861747983 2024-12-10T20:15:51,419 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/B in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,419 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6a0a708e5c0249339980b26dcfa57310, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e466074406c44ce38ba13adac4f22e4b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c7840d5426e94464b519f0ad0298aaa8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/1299d257efcf4f9593491ee754ca1d9d] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=46.4 K 2024-12-10T20:15:51,419 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 13cddb53b4194b2abb4983efe16c3789, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733861748704 2024-12-10T20:15:51,419 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0a708e5c0249339980b26dcfa57310, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733861746476 2024-12-10T20:15:51,420 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e466074406c44ce38ba13adac4f22e4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733861747289 2024-12-10T20:15:51,420 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7840d5426e94464b519f0ad0298aaa8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733861747983 2024-12-10T20:15:51,421 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1299d257efcf4f9593491ee754ca1d9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733861748704 2024-12-10T20:15:51,435 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:51,437 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#B#compaction#186 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:51,437 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121093b26c17992d414e8a62d53741e2afbe_fa895558d9876220a3ffb0096daff23e store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:51,438 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0e559d2a9e294dcd9db4916ba44b0571 is 50, key is test_row_0/B:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:51,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742051_1227 (size=13153) 2024-12-10T20:15:51,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T20:15:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,494 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T20:15:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:51,499 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121093b26c17992d414e8a62d53741e2afbe_fa895558d9876220a3ffb0096daff23e, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:51,499 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121093b26c17992d414e8a62d53741e2afbe_fa895558d9876220a3ffb0096daff23e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:51,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742052_1228 (size=4469) 2024-12-10T20:15:51,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121059c4255c6944459783eaa9dfdd32b1be_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861749906/Put/seqid=0 2024-12-10T20:15:51,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742053_1229 (size=9914) 2024-12-10T20:15:51,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:15:51,588 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121059c4255c6944459783eaa9dfdd32b1be_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121059c4255c6944459783eaa9dfdd32b1be_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:51,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/d5030908d4084350a452eae5c773e41a, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:15:51,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/d5030908d4084350a452eae5c773e41a is 175, key is test_row_0/A:col10/1733861749906/Put/seqid=0 2024-12-10T20:15:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742054_1230 (size=22561) 2024-12-10T20:15:51,905 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/0e559d2a9e294dcd9db4916ba44b0571 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0e559d2a9e294dcd9db4916ba44b0571 2024-12-10T20:15:51,924 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/B of fa895558d9876220a3ffb0096daff23e into 0e559d2a9e294dcd9db4916ba44b0571(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:51,924 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:51,924 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/B, priority=12, startTime=1733861751414; duration=0sec 2024-12-10T20:15:51,924 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:15:51,924 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:B 2024-12-10T20:15:51,924 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:15:51,928 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#A#compaction#187 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:15:51,928 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/b9174fb45a144adf9ac9c7bbec942e92 is 175, key is test_row_0/A:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:51,933 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:15:51,933 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): fa895558d9876220a3ffb0096daff23e/C is initiating minor compaction (all files) 2024-12-10T20:15:51,933 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa895558d9876220a3ffb0096daff23e/C in TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:51,934 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/3101da4a4aa943c08911f00a0fd5800f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/8d9ae587e5de43288d0783c1a1b2e3db, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7139838451c94362ac49a659bbf26a42] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp, totalSize=46.4 K 2024-12-10T20:15:51,934 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3101da4a4aa943c08911f00a0fd5800f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733861746476 2024-12-10T20:15:51,935 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9ae587e5de43288d0783c1a1b2e3db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733861747289 2024-12-10T20:15:51,935 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89dd6fc0ba274fc0a1a9f21fe1ab7d9c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1733861747983 2024-12-10T20:15:51,937 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7139838451c94362ac49a659bbf26a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733861748704 2024-12-10T20:15:51,953 DEBUG [regionserver/03a2ae042c95:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-10T20:15:51,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742055_1231 (size=32107) 2024-12-10T20:15:51,955 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa895558d9876220a3ffb0096daff23e#C#compaction#189 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:15:51,955 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/f9f270b0ac834b74a8695814813f8cbd is 50, key is test_row_0/C:col10/1733861748730/Put/seqid=0 2024-12-10T20:15:51,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742056_1232 (size=13153) 2024-12-10T20:15:52,002 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/f9f270b0ac834b74a8695814813f8cbd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f9f270b0ac834b74a8695814813f8cbd 2024-12-10T20:15:52,009 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=485, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/d5030908d4084350a452eae5c773e41a 2024-12-10T20:15:52,026 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/C of fa895558d9876220a3ffb0096daff23e into f9f270b0ac834b74a8695814813f8cbd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:52,026 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:52,026 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/C, priority=12, startTime=1733861751414; duration=0sec 2024-12-10T20:15:52,027 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:52,027 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:C 2024-12-10T20:15:52,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/cf4ae035a62645b0898a8e3bb2787d61 is 50, key is test_row_0/B:col10/1733861749906/Put/seqid=0 2024-12-10T20:15:52,069 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-12-10T20:15:52,069 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:52,069 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,069 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:52,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. as already flushing 2024-12-10T20:15:52,070 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-12-10T20:15:52,070 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:52,070 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,070 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742057_1233 (size=9857) 2024-12-10T20:15:52,078 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-12-10T20:15:52,078 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:52,078 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,079 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:52,088 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-12-10T20:15:52,088 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:52,088 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,088 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:52,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:52,359 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/b9174fb45a144adf9ac9c7bbec942e92 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b9174fb45a144adf9ac9c7bbec942e92 2024-12-10T20:15:52,378 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa895558d9876220a3ffb0096daff23e/A of fa895558d9876220a3ffb0096daff23e into b9174fb45a144adf9ac9c7bbec942e92(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:15:52,378 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:52,378 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., storeName=fa895558d9876220a3ffb0096daff23e/A, priority=12, startTime=1733861751414; duration=0sec 2024-12-10T20:15:52,378 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:15:52,378 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa895558d9876220a3ffb0096daff23e:A 2024-12-10T20:15:52,476 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/cf4ae035a62645b0898a8e3bb2787d61 2024-12-10T20:15:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/a8724748c75b450082bb84006b11e4d7 is 50, key is test_row_0/C:col10/1733861749906/Put/seqid=0 2024-12-10T20:15:52,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742058_1234 (size=9857) 2024-12-10T20:15:52,893 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/a8724748c75b450082bb84006b11e4d7 2024-12-10T20:15:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/d5030908d4084350a452eae5c773e41a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d5030908d4084350a452eae5c773e41a 2024-12-10T20:15:52,909 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d5030908d4084350a452eae5c773e41a, entries=100, sequenceid=485, filesize=22.0 K 2024-12-10T20:15:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/cf4ae035a62645b0898a8e3bb2787d61 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf4ae035a62645b0898a8e3bb2787d61 2024-12-10T20:15:52,918 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf4ae035a62645b0898a8e3bb2787d61, entries=100, sequenceid=485, filesize=9.6 K 2024-12-10T20:15:52,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/a8724748c75b450082bb84006b11e4d7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a8724748c75b450082bb84006b11e4d7 2024-12-10T20:15:52,924 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a8724748c75b450082bb84006b11e4d7, entries=100, sequenceid=485, filesize=9.6 K 2024-12-10T20:15:52,925 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=26.84 KB/27480 for fa895558d9876220a3ffb0096daff23e in 1430ms, sequenceid=485, compaction requested=false 2024-12-10T20:15:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for fa895558d9876220a3ffb0096daff23e: 2024-12-10T20:15:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-10T20:15:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-10T20:15:52,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-10T20:15:52,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8950 sec 2024-12-10T20:15:52,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9000 sec 2024-12-10T20:15:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T20:15:54,207 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:15:59,462 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-12-10T20:15:59,463 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:15:59,463 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:59,463 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:59,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2492 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2302 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1093 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3277 rows 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1118 2024-12-10T20:15:59,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3341 rows 2024-12-10T20:15:59,464 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:15:59,464 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:15:59,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:59,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:15:59,464 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:15:59,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T20:15:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T20:15:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T20:15:59,469 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861759469"}]},"ts":"1733861759469"} 2024-12-10T20:15:59,472 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T20:15:59,472 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T20:15:59,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:15:59,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, UNASSIGN}] 2024-12-10T20:15:59,475 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, UNASSIGN 2024-12-10T20:15:59,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:15:59,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, UNASSIGN because future has completed 2024-12-10T20:15:59,478 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:15:59,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:15:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T20:15:59,633 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close fa895558d9876220a3ffb0096daff23e 2024-12-10T20:15:59,633 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:15:59,633 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing fa895558d9876220a3ffb0096daff23e, disabling compactions & flushes 2024-12-10T20:15:59,634 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:59,634 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:59,634 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. after waiting 0 ms 2024-12-10T20:15:59,634 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:15:59,634 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing fa895558d9876220a3ffb0096daff23e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T20:15:59,635 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=A 2024-12-10T20:15:59,635 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:59,635 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=B 2024-12-10T20:15:59,635 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:59,635 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa895558d9876220a3ffb0096daff23e, store=C 2024-12-10T20:15:59,635 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:15:59,645 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103be615605c094ca2bc38dde8e0f3cb87_fa895558d9876220a3ffb0096daff23e is 50, key is test_row_0/A:col10/1733861759458/Put/seqid=0 2024-12-10T20:15:59,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742059_1235 (size=12454) 2024-12-10T20:15:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T20:16:00,051 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:00,061 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103be615605c094ca2bc38dde8e0f3cb87_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103be615605c094ca2bc38dde8e0f3cb87_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:00,063 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f31e571804224ea98c71e3da8cf3d01d, store: [table=TestAcidGuarantees family=A region=fa895558d9876220a3ffb0096daff23e] 2024-12-10T20:16:00,063 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f31e571804224ea98c71e3da8cf3d01d is 175, key is test_row_0/A:col10/1733861759458/Put/seqid=0 2024-12-10T20:16:00,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742060_1236 (size=31255) 2024-12-10T20:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T20:16:00,470 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=496, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f31e571804224ea98c71e3da8cf3d01d 2024-12-10T20:16:00,482 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6b332af145d84928a7d6b644d098de77 is 50, key is test_row_0/B:col10/1733861759458/Put/seqid=0 2024-12-10T20:16:00,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742061_1237 (size=12301) 2024-12-10T20:16:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T20:16:00,887 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6b332af145d84928a7d6b644d098de77 2024-12-10T20:16:00,895 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/eb3102042f3d418295e3278bbd134b19 is 50, key is test_row_0/C:col10/1733861759458/Put/seqid=0 2024-12-10T20:16:00,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742062_1238 (size=12301) 2024-12-10T20:16:01,300 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/eb3102042f3d418295e3278bbd134b19 2024-12-10T20:16:01,309 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/A/f31e571804224ea98c71e3da8cf3d01d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f31e571804224ea98c71e3da8cf3d01d 2024-12-10T20:16:01,314 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f31e571804224ea98c71e3da8cf3d01d, entries=150, sequenceid=496, filesize=30.5 K 2024-12-10T20:16:01,316 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/B/6b332af145d84928a7d6b644d098de77 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6b332af145d84928a7d6b644d098de77 2024-12-10T20:16:01,320 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6b332af145d84928a7d6b644d098de77, entries=150, sequenceid=496, filesize=12.0 K 2024-12-10T20:16:01,320 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/.tmp/C/eb3102042f3d418295e3278bbd134b19 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/eb3102042f3d418295e3278bbd134b19 2024-12-10T20:16:01,325 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/eb3102042f3d418295e3278bbd134b19, entries=150, sequenceid=496, filesize=12.0 K 2024-12-10T20:16:01,325 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fa895558d9876220a3ffb0096daff23e in 1691ms, sequenceid=496, compaction requested=true 2024-12-10T20:16:01,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f960a9d32e5b47998bdaf3d0b8226cdc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a535bc0bf4a5468e8e3bc78adf2806e0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0cbb283536b84b10ae71ab9dd7c42199, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/92ca4d450ba74955b5863840208d3403, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d381edd21b034ff7ad0c6ca4739d9c4c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f626054b47a946d68cdeca0eb1c72570, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789] to archive 2024-12-10T20:16:01,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:01,330 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f2b23ecd1e1f4bfc93e75d88561c0800 2024-12-10T20:16:01,330 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/cb2b0b5f2c7143e0a65f5e1443581368 2024-12-10T20:16:01,330 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f960a9d32e5b47998bdaf3d0b8226cdc to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f960a9d32e5b47998bdaf3d0b8226cdc 2024-12-10T20:16:01,330 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/7037b3c3335343e4a241d83b1b10d09a 2024-12-10T20:16:01,331 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/4e3f01c9814049e698e03895889d0c36 2024-12-10T20:16:01,331 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/8ad031c3690d496ebdf874569684d135 2024-12-10T20:16:01,331 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0a9b3fd9e4ac4bcab478b83608bc35cd 2024-12-10T20:16:01,332 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a535bc0bf4a5468e8e3bc78adf2806e0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a535bc0bf4a5468e8e3bc78adf2806e0 2024-12-10T20:16:01,333 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/848a0b57902249719b717bf9a27071c5 2024-12-10T20:16:01,333 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0cbb283536b84b10ae71ab9dd7c42199 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/0cbb283536b84b10ae71ab9dd7c42199 2024-12-10T20:16:01,334 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/a8e72042d5ff4f059f93f9f5a381599f 2024-12-10T20:16:01,335 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f08a25cbbab146e29a7cac3686fdd520 2024-12-10T20:16:01,335 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/ce1a46d30f6347ffafa4c03acd365194 2024-12-10T20:16:01,335 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b88a134cd70a4ac397a102e2c1ee69e3 2024-12-10T20:16:01,335 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/466987f21fd34b93b8c88ebb48c45a15 2024-12-10T20:16:01,336 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/92ca4d450ba74955b5863840208d3403 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/92ca4d450ba74955b5863840208d3403 2024-12-10T20:16:01,336 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/aa4e7134f5b84b97bccc0d7c77d3000e 2024-12-10T20:16:01,337 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/811e7dd1772e43d3ab315837f7e285a9 2024-12-10T20:16:01,338 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d381edd21b034ff7ad0c6ca4739d9c4c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d381edd21b034ff7ad0c6ca4739d9c4c 2024-12-10T20:16:01,337 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/65eb720f50694fcdbd7d5bde7cbe8260 2024-12-10T20:16:01,338 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/93b435bfd277484583fb47d10ace838e 2024-12-10T20:16:01,338 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f626054b47a946d68cdeca0eb1c72570 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f626054b47a946d68cdeca0eb1c72570 2024-12-10T20:16:01,338 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/13cddb53b4194b2abb4983efe16c3789 2024-12-10T20:16:01,338 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/2c9a0eb87d6e43c7b6e452d6a8017dc5 2024-12-10T20:16:01,338 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f16748b64a0748209eaf9d7964a287df 2024-12-10T20:16:01,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6f9a7f37bae94164be67a0843cb933d4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/91741b57199f491a8e370a4839d6e61e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e34e95bdcb7c4996befef1c5a9c3bf75, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/3a82ba85c03141bcb6218e9ec9d75d08, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5b56f9f66e39449b85cdd47c6116cf83, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7c8ec1b86dc2498f9e8a17f31ad0313d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0636a3070a7948f7be3b53001ec54e19, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/87fcc0d3a79f4680b6babcf25122e44b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/511f15a9e3194f1a93d73c4e20aba7e3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c8dcfdef15104210b77500c5ee564c1a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/b4a2afb6e059462d94395938211863d7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/817d54e5b31f437a89c1d2e47741d6b1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0b330595ccaf48edb2ee0e29fe94ac35, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf2d2916845e4acea8d04905a26c2c3e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/94d984e53070496187d238072c574f8e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5909b2e1238f472d91744d90c92a606b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/676ea811968a46bbbcd88aef292f9201, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/15455c785a4a41629cf2e30d5fb12f4e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/2f5efaa7f1b241ac96e9e5258af93bb9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7f64e8492978454591eef72e97dccd1d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6a0a708e5c0249339980b26dcfa57310, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6847c0ac1f56463e8121cb9156aef1af, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e466074406c44ce38ba13adac4f22e4b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c7840d5426e94464b519f0ad0298aaa8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/1299d257efcf4f9593491ee754ca1d9d] to archive 2024-12-10T20:16:01,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e34e95bdcb7c4996befef1c5a9c3bf75 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e34e95bdcb7c4996befef1c5a9c3bf75 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/91741b57199f491a8e370a4839d6e61e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/91741b57199f491a8e370a4839d6e61e 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/87fcc0d3a79f4680b6babcf25122e44b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/87fcc0d3a79f4680b6babcf25122e44b 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7c8ec1b86dc2498f9e8a17f31ad0313d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7c8ec1b86dc2498f9e8a17f31ad0313d 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5b56f9f66e39449b85cdd47c6116cf83 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5b56f9f66e39449b85cdd47c6116cf83 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/3a82ba85c03141bcb6218e9ec9d75d08 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/3a82ba85c03141bcb6218e9ec9d75d08 2024-12-10T20:16:01,343 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0636a3070a7948f7be3b53001ec54e19 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0636a3070a7948f7be3b53001ec54e19 2024-12-10T20:16:01,344 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c8dcfdef15104210b77500c5ee564c1a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c8dcfdef15104210b77500c5ee564c1a 2024-12-10T20:16:01,344 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/511f15a9e3194f1a93d73c4e20aba7e3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/511f15a9e3194f1a93d73c4e20aba7e3 2024-12-10T20:16:01,345 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/b4a2afb6e059462d94395938211863d7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/b4a2afb6e059462d94395938211863d7 2024-12-10T20:16:01,345 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/817d54e5b31f437a89c1d2e47741d6b1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/817d54e5b31f437a89c1d2e47741d6b1 2024-12-10T20:16:01,345 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf2d2916845e4acea8d04905a26c2c3e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf2d2916845e4acea8d04905a26c2c3e 2024-12-10T20:16:01,345 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/94d984e53070496187d238072c574f8e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/94d984e53070496187d238072c574f8e 2024-12-10T20:16:01,345 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0b330595ccaf48edb2ee0e29fe94ac35 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0b330595ccaf48edb2ee0e29fe94ac35 2024-12-10T20:16:01,346 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5909b2e1238f472d91744d90c92a606b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/5909b2e1238f472d91744d90c92a606b 2024-12-10T20:16:01,346 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/676ea811968a46bbbcd88aef292f9201 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/676ea811968a46bbbcd88aef292f9201 2024-12-10T20:16:01,347 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/2f5efaa7f1b241ac96e9e5258af93bb9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/2f5efaa7f1b241ac96e9e5258af93bb9 2024-12-10T20:16:01,347 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7f64e8492978454591eef72e97dccd1d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/7f64e8492978454591eef72e97dccd1d 2024-12-10T20:16:01,347 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6f9a7f37bae94164be67a0843cb933d4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6f9a7f37bae94164be67a0843cb933d4 2024-12-10T20:16:01,347 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6a0a708e5c0249339980b26dcfa57310 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6a0a708e5c0249339980b26dcfa57310 2024-12-10T20:16:01,347 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6847c0ac1f56463e8121cb9156aef1af to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6847c0ac1f56463e8121cb9156aef1af 2024-12-10T20:16:01,347 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/15455c785a4a41629cf2e30d5fb12f4e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/15455c785a4a41629cf2e30d5fb12f4e 2024-12-10T20:16:01,348 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c7840d5426e94464b519f0ad0298aaa8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/c7840d5426e94464b519f0ad0298aaa8 2024-12-10T20:16:01,348 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e466074406c44ce38ba13adac4f22e4b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/e466074406c44ce38ba13adac4f22e4b 2024-12-10T20:16:01,348 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/1299d257efcf4f9593491ee754ca1d9d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/1299d257efcf4f9593491ee754ca1d9d 2024-12-10T20:16:01,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a1306fe8b7a14c94833bb7b2d980da78, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2b89c8b5fdb04ed7b95167932a794450, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7b165492d7c64c918f172e9265d602f3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e334b32c942c4fd1bbcb49dd775b5fba, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/da8b101d5ea54df39cff213fa597fcab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/0d3a6b978b614fb7bfced8dd57a7691e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/b26a9264945e4ed5bf84a05215857064, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6b200d8f835a48839d2bcc136acdb293, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5de484fe043c48e4ab42b782210201d8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2da084147ed74ba4ad10402f55ffa949, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/9e08e3caceb948a9ac1b950f0801de0a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/1d307ecafb204592b271b511425a124e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/26033eae457e43fa9a8cb2b9bec68d32, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e009a08c619e45fdb57cf3722ee1f768, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/ff9d623bd44d47b2bb4871f382ee4295, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f699b798d23a457aa3c999379af67b0f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/aece297414964b5abdc4437a7eb51b02, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6a2f500a7e44456796bd16a5aa06f37e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5c1c6963f0cc41a1888db691afaeda85, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/4df18eb858a84aeba802527d0143fac8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/3101da4a4aa943c08911f00a0fd5800f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7f706c4329954d0c81551dc9379c6cab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/8d9ae587e5de43288d0783c1a1b2e3db, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7139838451c94362ac49a659bbf26a42] to archive 2024-12-10T20:16:01,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:01,351 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2b89c8b5fdb04ed7b95167932a794450 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2b89c8b5fdb04ed7b95167932a794450 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a1306fe8b7a14c94833bb7b2d980da78 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a1306fe8b7a14c94833bb7b2d980da78 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/0d3a6b978b614fb7bfced8dd57a7691e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/0d3a6b978b614fb7bfced8dd57a7691e 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e334b32c942c4fd1bbcb49dd775b5fba to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e334b32c942c4fd1bbcb49dd775b5fba 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7b165492d7c64c918f172e9265d602f3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7b165492d7c64c918f172e9265d602f3 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/da8b101d5ea54df39cff213fa597fcab to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/da8b101d5ea54df39cff213fa597fcab 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/b26a9264945e4ed5bf84a05215857064 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/b26a9264945e4ed5bf84a05215857064 2024-12-10T20:16:01,352 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6b200d8f835a48839d2bcc136acdb293 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6b200d8f835a48839d2bcc136acdb293 2024-12-10T20:16:01,353 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5de484fe043c48e4ab42b782210201d8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5de484fe043c48e4ab42b782210201d8 2024-12-10T20:16:01,354 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/9e08e3caceb948a9ac1b950f0801de0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/9e08e3caceb948a9ac1b950f0801de0a 2024-12-10T20:16:01,354 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e009a08c619e45fdb57cf3722ee1f768 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/e009a08c619e45fdb57cf3722ee1f768 2024-12-10T20:16:01,354 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2da084147ed74ba4ad10402f55ffa949 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/2da084147ed74ba4ad10402f55ffa949 2024-12-10T20:16:01,354 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/26033eae457e43fa9a8cb2b9bec68d32 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/26033eae457e43fa9a8cb2b9bec68d32 2024-12-10T20:16:01,354 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/1d307ecafb204592b271b511425a124e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/1d307ecafb204592b271b511425a124e 2024-12-10T20:16:01,354 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f699b798d23a457aa3c999379af67b0f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f699b798d23a457aa3c999379af67b0f 2024-12-10T20:16:01,355 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/ff9d623bd44d47b2bb4871f382ee4295 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/ff9d623bd44d47b2bb4871f382ee4295 2024-12-10T20:16:01,355 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/aece297414964b5abdc4437a7eb51b02 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/aece297414964b5abdc4437a7eb51b02 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6a2f500a7e44456796bd16a5aa06f37e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/6a2f500a7e44456796bd16a5aa06f37e 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/4df18eb858a84aeba802527d0143fac8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/4df18eb858a84aeba802527d0143fac8 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5c1c6963f0cc41a1888db691afaeda85 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/5c1c6963f0cc41a1888db691afaeda85 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/3101da4a4aa943c08911f00a0fd5800f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/3101da4a4aa943c08911f00a0fd5800f 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7f706c4329954d0c81551dc9379c6cab to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7f706c4329954d0c81551dc9379c6cab 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/8d9ae587e5de43288d0783c1a1b2e3db to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/8d9ae587e5de43288d0783c1a1b2e3db 2024-12-10T20:16:01,356 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/89dd6fc0ba274fc0a1a9f21fe1ab7d9c 2024-12-10T20:16:01,357 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7139838451c94362ac49a659bbf26a42 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/7139838451c94362ac49a659bbf26a42 2024-12-10T20:16:01,361 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/recovered.edits/499.seqid, newMaxSeqId=499, maxSeqId=4 2024-12-10T20:16:01,363 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. 2024-12-10T20:16:01,363 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for fa895558d9876220a3ffb0096daff23e: Waiting for close lock at 1733861759633Running coprocessor pre-close hooks at 1733861759633Disabling compacts and flushes for region at 1733861759633Disabling writes for close at 1733861759634 (+1 ms)Obtaining lock to block concurrent updates at 1733861759634Preparing flush snapshotting stores in fa895558d9876220a3ffb0096daff23e at 1733861759634Finished memstore snapshotting TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733861759636 (+2 ms)Flushing stores of TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e. at 1733861759637 (+1 ms)Flushing fa895558d9876220a3ffb0096daff23e/A: creating writer at 1733861759637Flushing mob file fa895558d9876220a3ffb0096daff23e/A: appending metadata at 1733861759645 (+8 ms)Flushing mob file fa895558d9876220a3ffb0096daff23e/A: closing flushed file at 1733861759645Flushing fa895558d9876220a3ffb0096daff23e/A: appending metadata at 1733861760063 (+418 ms)Flushing fa895558d9876220a3ffb0096daff23e/A: closing flushed file at 1733861760063Flushing fa895558d9876220a3ffb0096daff23e/B: creating writer at 1733861760477 (+414 ms)Flushing fa895558d9876220a3ffb0096daff23e/B: appending metadata at 1733861760481 (+4 ms)Flushing fa895558d9876220a3ffb0096daff23e/B: closing flushed file at 1733861760481Flushing fa895558d9876220a3ffb0096daff23e/C: creating writer at 1733861760892 (+411 ms)Flushing fa895558d9876220a3ffb0096daff23e/C: appending metadata at 1733861760895 (+3 ms)Flushing fa895558d9876220a3ffb0096daff23e/C: closing flushed file at 1733861760895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a0a886: reopening flushed file at 1733861761308 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cee8c03: reopening flushed file at 1733861761315 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72996f04: reopening flushed file at 1733861761320 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fa895558d9876220a3ffb0096daff23e in 1691ms, sequenceid=496, compaction requested=true at 1733861761325 (+5 ms)Writing region close event to WAL at 1733861761358 (+33 ms)Running coprocessor post-close hooks at 1733861761362 (+4 ms)Closed at 1733861761362 2024-12-10T20:16:01,365 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,365 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=fa895558d9876220a3ffb0096daff23e, regionState=CLOSED 2024-12-10T20:16:01,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:16:01,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-10T20:16:01,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure fa895558d9876220a3ffb0096daff23e, server=03a2ae042c95,38133,1733861689839 in 1.8880 sec 2024-12-10T20:16:01,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-10T20:16:01,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa895558d9876220a3ffb0096daff23e, UNASSIGN in 1.8950 sec 2024-12-10T20:16:01,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-10T20:16:01,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8990 sec 2024-12-10T20:16:01,373 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861761373"}]},"ts":"1733861761373"} 2024-12-10T20:16:01,374 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T20:16:01,374 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T20:16:01,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9090 sec 2024-12-10T20:16:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T20:16:01,618 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:01,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T20:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,624 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T20:16:01,625 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,630 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,634 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/recovered.edits] 2024-12-10T20:16:01,640 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b9174fb45a144adf9ac9c7bbec942e92 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/b9174fb45a144adf9ac9c7bbec942e92 2024-12-10T20:16:01,640 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f31e571804224ea98c71e3da8cf3d01d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/f31e571804224ea98c71e3da8cf3d01d 2024-12-10T20:16:01,641 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d5030908d4084350a452eae5c773e41a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/A/d5030908d4084350a452eae5c773e41a 2024-12-10T20:16:01,643 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf4ae035a62645b0898a8e3bb2787d61 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/cf4ae035a62645b0898a8e3bb2787d61 2024-12-10T20:16:01,643 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0e559d2a9e294dcd9db4916ba44b0571 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/0e559d2a9e294dcd9db4916ba44b0571 2024-12-10T20:16:01,643 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6b332af145d84928a7d6b644d098de77 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/B/6b332af145d84928a7d6b644d098de77 2024-12-10T20:16:01,646 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a8724748c75b450082bb84006b11e4d7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/a8724748c75b450082bb84006b11e4d7 2024-12-10T20:16:01,646 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/eb3102042f3d418295e3278bbd134b19 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/eb3102042f3d418295e3278bbd134b19 2024-12-10T20:16:01,646 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f9f270b0ac834b74a8695814813f8cbd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/C/f9f270b0ac834b74a8695814813f8cbd 2024-12-10T20:16:01,648 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/recovered.edits/499.seqid to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e/recovered.edits/499.seqid 2024-12-10T20:16:01,649 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,649 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T20:16:01,649 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T20:16:01,650 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121029378202cb714249bed0897834813cac_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121029378202cb714249bed0897834813cac_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121002b6b4806723466da9192dc821667ae9_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121002b6b4806723466da9192dc821667ae9_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210451522195d04476fb295679b072be042_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210451522195d04476fb295679b072be042_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103c2490bb11874ff1844817ac43aa91e1_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103c2490bb11874ff1844817ac43aa91e1_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102eba0865e61d4ffd83aac6a450778749_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102eba0865e61d4ffd83aac6a450778749_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101b35ac6f7a8143518b2455bb043a3524_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101b35ac6f7a8143518b2455bb043a3524_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103be615605c094ca2bc38dde8e0f3cb87_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103be615605c094ca2bc38dde8e0f3cb87_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,658 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121043ee902bc34e4384b5f593ee94c818c2_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121043ee902bc34e4384b5f593ee94c818c2_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,659 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121059c4255c6944459783eaa9dfdd32b1be_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121059c4255c6944459783eaa9dfdd32b1be_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105b6e8e32f49646c5a529204a864fc054_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105b6e8e32f49646c5a529204a864fc054_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106c0f0a2435674e82849020c04f3d581e_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106c0f0a2435674e82849020c04f3d581e_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f63001f9c924515a0f154f598bf2926_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108f63001f9c924515a0f154f598bf2926_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121085f40c7685224ad48234645bdb82fd6b_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121085f40c7685224ad48234645bdb82fd6b_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105ac157c5cc684b7bbc8cd86d71647118_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105ac157c5cc684b7bbc8cd86d71647118_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109cce29007cbc492ebd19d830192dde8d_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109cce29007cbc492ebd19d830192dde8d_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,660 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121060411d901b614c7fb45cf0b412c6b4cc_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121060411d901b614c7fb45cf0b412c6b4cc_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,661 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a3cc322581e74ac0a30b8cb5c2873ca7_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a3cc322581e74ac0a30b8cb5c2873ca7_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,661 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210acd77cf5e26440288ea17823447acbee_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210acd77cf5e26440288ea17823447acbee_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,661 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c64c6d23f23542eaa48e4dd2ac1bf419_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c64c6d23f23542eaa48e4dd2ac1bf419_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,661 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e86bda3c33df4674b9575692f71c0fe1_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e86bda3c33df4674b9575692f71c0fe1_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,661 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fe77e024c83d40c3a3e7ec87cc157ab0_fa895558d9876220a3ffb0096daff23e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fe77e024c83d40c3a3e7ec87cc157ab0_fa895558d9876220a3ffb0096daff23e 2024-12-10T20:16:01,662 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T20:16:01,664 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,666 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T20:16:01,668 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T20:16:01,670 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,670 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T20:16:01,670 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733861761670"}]},"ts":"9223372036854775807"} 2024-12-10T20:16:01,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T20:16:01,674 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fa895558d9876220a3ffb0096daff23e, NAME => 'TestAcidGuarantees,,1733861727798.fa895558d9876220a3ffb0096daff23e.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T20:16:01,674 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T20:16:01,674 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733861761674"}]},"ts":"9223372036854775807"} 2024-12-10T20:16:01,676 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T20:16:01,677 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-12-10T20:16:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T20:16:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T20:16:01,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:01,741 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=253 (was 251) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-12 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-15 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-13 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-16 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-14 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1060333068_22 at /127.0.0.1:44944 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=848 (was 724) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=867 (was 248) - AvailableMemoryMB LEAK? - 2024-12-10T20:16:01,748 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=253, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=848, ProcessCount=11, AvailableMemoryMB=867 2024-12-10T20:16:01,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:16:01,749 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:16:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:01,751 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:16:01,751 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:01,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-10T20:16:01,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:16:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-10T20:16:01,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742063_1239 (size=1061) 2024-12-10T20:16:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-10T20:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-10T20:16:02,160 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:16:02,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742064_1240 (size=53) 2024-12-10T20:16:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-10T20:16:02,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:16:02,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d574f3055e0a5aa98dce8c43f5c73a8d, disabling compactions & flushes 2024-12-10T20:16:02,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. after waiting 0 ms 2024-12-10T20:16:02,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,566 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d574f3055e0a5aa98dce8c43f5c73a8d: Waiting for close lock at 1733861762566Disabling compacts and flushes for region at 1733861762566Disabling writes for close at 1733861762566Writing region close event to WAL at 1733861762566Closed at 1733861762566 2024-12-10T20:16:02,568 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:16:02,568 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733861762568"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733861762568"}]},"ts":"1733861762568"} 2024-12-10T20:16:02,570 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:16:02,571 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:16:02,571 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861762571"}]},"ts":"1733861762571"} 2024-12-10T20:16:02,573 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T20:16:02,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, ASSIGN}] 2024-12-10T20:16:02,575 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, ASSIGN 2024-12-10T20:16:02,575 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, ASSIGN; state=OFFLINE, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=false 2024-12-10T20:16:02,726 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d574f3055e0a5aa98dce8c43f5c73a8d, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:02,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, ASSIGN because future has completed 2024-12-10T20:16:02,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:16:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-10T20:16:02,888 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,889 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:16:02,889 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,890 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:16:02,890 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,890 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,893 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,894 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:02,894 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d574f3055e0a5aa98dce8c43f5c73a8d columnFamilyName A 2024-12-10T20:16:02,895 DEBUG [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:02,895 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.HStore(327): Store=d574f3055e0a5aa98dce8c43f5c73a8d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:02,895 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,896 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:02,896 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d574f3055e0a5aa98dce8c43f5c73a8d columnFamilyName B 2024-12-10T20:16:02,896 DEBUG [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:02,897 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.HStore(327): Store=d574f3055e0a5aa98dce8c43f5c73a8d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:02,897 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,898 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:02,898 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d574f3055e0a5aa98dce8c43f5c73a8d columnFamilyName C 2024-12-10T20:16:02,898 DEBUG [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:02,898 INFO [StoreOpener-d574f3055e0a5aa98dce8c43f5c73a8d-1 {}] regionserver.HStore(327): Store=d574f3055e0a5aa98dce8c43f5c73a8d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:02,899 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,899 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,899 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,900 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,901 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,901 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,901 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:16:02,902 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,904 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:16:02,905 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened d574f3055e0a5aa98dce8c43f5c73a8d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64876238, jitterRate=-0.03326871991157532}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:16:02,905 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:02,905 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for d574f3055e0a5aa98dce8c43f5c73a8d: Running coprocessor pre-open hook at 1733861762890Writing region info on filesystem at 1733861762890Initializing all the Stores at 1733861762893 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861762893Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861762893Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861762893Cleaning up temporary data from old regions at 1733861762901 (+8 ms)Running coprocessor post-open hooks at 1733861762905 (+4 ms)Region opened successfully at 1733861762905 2024-12-10T20:16:02,906 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., pid=61, masterSystemTime=1733861762881 2024-12-10T20:16:02,908 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,908 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:02,908 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d574f3055e0a5aa98dce8c43f5c73a8d, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:02,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:16:02,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-10T20:16:02,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 in 182 msec 2024-12-10T20:16:02,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-10T20:16:02,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, ASSIGN in 339 msec 2024-12-10T20:16:02,915 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:16:02,916 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861762915"}]},"ts":"1733861762915"} 2024-12-10T20:16:02,917 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T20:16:02,918 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:16:02,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-12-10T20:16:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-10T20:16:03,897 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:03,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7642a08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,899 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178ca358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633bd603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,906 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:16:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:16:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42e25552 2024-12-10T20:16:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:16:03,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:16:03,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0f448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,910 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1974dfe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,912 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb79441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d560643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,921 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59b38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,923 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a5ee6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6509bdf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,930 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26550b73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c250c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b646580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,936 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74858e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,938 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c082a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c85ada2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,941 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50941e19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,943 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393b59f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829bcb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178a8a7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d5f456b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d96952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,954 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be29939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4274e63d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24595b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,959 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b58a0a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,961 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14741f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c58d7ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,964 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2829b3ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,966 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a755844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12358372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:03,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:03,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d997e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:03,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450da936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:03,977 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,977 DEBUG [Thread-1127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,977 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,977 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:03,989 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,989 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,990 DEBUG [Thread-1136 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,990 DEBUG [Thread-1136 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:03,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,992 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-10T20:16:03,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,994 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:03,995 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:03,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:16:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:03,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:04,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:04,001 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:04,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:04,005 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:04,006 DEBUG [Thread-1138 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:04,006 DEBUG [Thread-1138 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:04,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:04,020 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:04,021 DEBUG [Thread-1132 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:04,021 DEBUG [Thread-1132 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:04,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:16:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861824038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861824045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f363aee6d03841be985d37d364eb53f7 is 50, key is test_row_0/A:col10/1733861763988/Put/seqid=0 2024-12-10T20:16:04,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861824049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733861764022, max timestamp=1733861764045Num uniques 859; 2024-12-10T20:16:04,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T20:16:04,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733861764022, max timestamp=1733861764045Num uniques 859; 2024-12-10T20:16:04,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T20:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733861764022, max timestamp=1733861764045Num uniques 859; 2024-12-10T20:16:04,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T20:16:04,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742065_1241 (size=12001) 2024-12-10T20:16:04,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f363aee6d03841be985d37d364eb53f7 2024-12-10T20:16:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:04,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d572c60557f64e9ab67c735872727b2b is 50, key is test_row_0/B:col10/1733861763988/Put/seqid=0 2024-12-10T20:16:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861824138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861824141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742066_1242 (size=12001) 2024-12-10T20:16:04,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d572c60557f64e9ab67c735872727b2b 2024-12-10T20:16:04,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861824169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861824171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861824160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/66e154f1147c4507ae03aa7f24947242 is 50, key is test_row_0/C:col10/1733861763988/Put/seqid=0 2024-12-10T20:16:04,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742067_1243 (size=12001) 2024-12-10T20:16:04,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861824247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861824264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:04,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861824377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861824389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861824393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861824457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:04,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:04,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861824477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/66e154f1147c4507ae03aa7f24947242 2024-12-10T20:16:04,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f363aee6d03841be985d37d364eb53f7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f363aee6d03841be985d37d364eb53f7 2024-12-10T20:16:04,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f363aee6d03841be985d37d364eb53f7, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T20:16:04,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d572c60557f64e9ab67c735872727b2b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d572c60557f64e9ab67c735872727b2b 2024-12-10T20:16:04,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d572c60557f64e9ab67c735872727b2b, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T20:16:04,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/66e154f1147c4507ae03aa7f24947242 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/66e154f1147c4507ae03aa7f24947242 2024-12-10T20:16:04,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/66e154f1147c4507ae03aa7f24947242, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T20:16:04,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.28 KB/129315 for d574f3055e0a5aa98dce8c43f5c73a8d in 694ms, sequenceid=12, compaction requested=false 2024-12-10T20:16:04,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:04,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-12-10T20:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:04,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/c730f0d968f749478c7e515ee6daba09 is 50, key is test_row_0/A:col10/1733861764690/Put/seqid=0 2024-12-10T20:16:04,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861824727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861824728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861824733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742068_1244 (size=14341) 2024-12-10T20:16:04,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861824766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861824787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861824846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861824847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861824847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:04,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:04,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:04,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:04,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861825057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861825057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861825060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:05,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/c730f0d968f749478c7e515ee6daba09 2024-12-10T20:16:05,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/192639d35e53466ca0d723dc7954b734 is 50, key is test_row_0/B:col10/1733861764690/Put/seqid=0 2024-12-10T20:16:05,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742069_1245 (size=12001) 2024-12-10T20:16:05,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/192639d35e53466ca0d723dc7954b734 2024-12-10T20:16:05,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ff5373c272334f0da27722834bbef14f is 50, key is test_row_0/C:col10/1733861764690/Put/seqid=0 2024-12-10T20:16:05,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742070_1246 (size=12001) 2024-12-10T20:16:05,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ff5373c272334f0da27722834bbef14f 2024-12-10T20:16:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/c730f0d968f749478c7e515ee6daba09 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/c730f0d968f749478c7e515ee6daba09 2024-12-10T20:16:05,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/c730f0d968f749478c7e515ee6daba09, entries=200, sequenceid=49, filesize=14.0 K 2024-12-10T20:16:05,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/192639d35e53466ca0d723dc7954b734 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/192639d35e53466ca0d723dc7954b734 2024-12-10T20:16:05,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/192639d35e53466ca0d723dc7954b734, entries=150, sequenceid=49, filesize=11.7 K 2024-12-10T20:16:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ff5373c272334f0da27722834bbef14f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ff5373c272334f0da27722834bbef14f 2024-12-10T20:16:05,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ff5373c272334f0da27722834bbef14f, entries=150, sequenceid=49, filesize=11.7 K 2024-12-10T20:16:05,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~132.99 KB/136185, heapSize ~382.20 KB/391368, currentSize=53.67 KB/54960 for d574f3055e0a5aa98dce8c43f5c73a8d in 576ms, sequenceid=49, compaction requested=false 2024-12-10T20:16:05,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:05,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T20:16:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:05,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:05,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/cb6d8a0fe2734826a321e95ac1193788 is 50, key is test_row_0/A:col10/1733861764722/Put/seqid=0 2024-12-10T20:16:05,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742071_1247 (size=14341) 2024-12-10T20:16:05,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/cb6d8a0fe2734826a321e95ac1193788 2024-12-10T20:16:05,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d44741011557479495e7d2bae714ad53 is 50, key is test_row_0/B:col10/1733861764722/Put/seqid=0 2024-12-10T20:16:05,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861825353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861825356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742072_1248 (size=12001) 2024-12-10T20:16:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861825368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861825368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d44741011557479495e7d2bae714ad53 2024-12-10T20:16:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861825380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/8b341a5735aa49739278c563fc8e0e06 is 50, key is test_row_0/C:col10/1733861764722/Put/seqid=0 2024-12-10T20:16:05,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:05,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742073_1249 (size=12001) 2024-12-10T20:16:05,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/8b341a5735aa49739278c563fc8e0e06 2024-12-10T20:16:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861825468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861825468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/cb6d8a0fe2734826a321e95ac1193788 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cb6d8a0fe2734826a321e95ac1193788 2024-12-10T20:16:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cb6d8a0fe2734826a321e95ac1193788, entries=200, sequenceid=61, filesize=14.0 K 2024-12-10T20:16:05,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d44741011557479495e7d2bae714ad53 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d44741011557479495e7d2bae714ad53 2024-12-10T20:16:05,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d44741011557479495e7d2bae714ad53, entries=150, sequenceid=61, filesize=11.7 K 2024-12-10T20:16:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/8b341a5735aa49739278c563fc8e0e06 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/8b341a5735aa49739278c563fc8e0e06 2024-12-10T20:16:05,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/8b341a5735aa49739278c563fc8e0e06, entries=150, sequenceid=61, filesize=11.7 K 2024-12-10T20:16:05,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d574f3055e0a5aa98dce8c43f5c73a8d in 249ms, sequenceid=61, compaction requested=true 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:05,543 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:16:05,543 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:05,544 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:05,544 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:05,544 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,545 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f363aee6d03841be985d37d364eb53f7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/c730f0d968f749478c7e515ee6daba09, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cb6d8a0fe2734826a321e95ac1193788] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=39.7 K 2024-12-10T20:16:05,545 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:05,545 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:05,545 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,545 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d572c60557f64e9ab67c735872727b2b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/192639d35e53466ca0d723dc7954b734, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d44741011557479495e7d2bae714ad53] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.2 K 2024-12-10T20:16:05,545 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f363aee6d03841be985d37d364eb53f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733861763988 2024-12-10T20:16:05,546 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d572c60557f64e9ab67c735872727b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733861763988 2024-12-10T20:16:05,546 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 192639d35e53466ca0d723dc7954b734, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733861764085 2024-12-10T20:16:05,546 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c730f0d968f749478c7e515ee6daba09, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733861764085 2024-12-10T20:16:05,546 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb6d8a0fe2734826a321e95ac1193788, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733861764720 2024-12-10T20:16:05,546 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d44741011557479495e7d2bae714ad53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733861764722 2024-12-10T20:16:05,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:05,563 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:05,574 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:05,575 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/6448ae8462f8403c82e6f4a0f7551ac4 is 50, key is test_row_0/B:col10/1733861764722/Put/seqid=0 2024-12-10T20:16:05,585 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:05,586 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/dc653801d6bc4322b051ede096c0d62f is 50, key is test_row_0/A:col10/1733861764722/Put/seqid=0 2024-12-10T20:16:05,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/73fe05a11c124c95a493448b04b4ff72 is 50, key is test_row_0/A:col10/1733861765324/Put/seqid=0 2024-12-10T20:16:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742074_1250 (size=12104) 2024-12-10T20:16:05,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742075_1251 (size=12104) 2024-12-10T20:16:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:05,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861825729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742076_1252 (size=12001) 2024-12-10T20:16:05,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861825730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861825837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861825846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861825877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861825877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861825897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861826047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,060 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/6448ae8462f8403c82e6f4a0f7551ac4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6448ae8462f8403c82e6f4a0f7551ac4 2024-12-10T20:16:06,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861826058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,073 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 6448ae8462f8403c82e6f4a0f7551ac4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:06,073 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/dc653801d6bc4322b051ede096c0d62f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/dc653801d6bc4322b051ede096c0d62f 2024-12-10T20:16:06,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:06,073 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861765543; duration=0sec 2024-12-10T20:16:06,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:06,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:06,073 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:06,079 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:06,080 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:06,080 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:06,080 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/66e154f1147c4507ae03aa7f24947242, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ff5373c272334f0da27722834bbef14f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/8b341a5735aa49739278c563fc8e0e06] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.2 K 2024-12-10T20:16:06,082 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e154f1147c4507ae03aa7f24947242, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733861763988 2024-12-10T20:16:06,083 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ff5373c272334f0da27722834bbef14f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733861764085 2024-12-10T20:16:06,084 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b341a5735aa49739278c563fc8e0e06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733861764722 2024-12-10T20:16:06,087 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into dc653801d6bc4322b051ede096c0d62f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:06,087 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:06,087 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861765543; duration=0sec 2024-12-10T20:16:06,087 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:06,087 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:06,099 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:06,100 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/824fffaf6c5b481f82d0afb04cc14c6e is 50, key is test_row_0/C:col10/1733861764722/Put/seqid=0 2024-12-10T20:16:06,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742077_1253 (size=12104) 2024-12-10T20:16:06,138 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/73fe05a11c124c95a493448b04b4ff72 2024-12-10T20:16:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:06,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/8b5ec2253e34439a8bbf8d05cdff2e41 is 50, key is test_row_0/B:col10/1733861765324/Put/seqid=0 2024-12-10T20:16:06,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742078_1254 (size=12001) 2024-12-10T20:16:06,198 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/8b5ec2253e34439a8bbf8d05cdff2e41 2024-12-10T20:16:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c8a67a494bbd49eb8093ac3b3bbb3d02 is 50, key is test_row_0/C:col10/1733861765324/Put/seqid=0 2024-12-10T20:16:06,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742079_1255 (size=12001) 2024-12-10T20:16:06,277 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c8a67a494bbd49eb8093ac3b3bbb3d02 2024-12-10T20:16:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/73fe05a11c124c95a493448b04b4ff72 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/73fe05a11c124c95a493448b04b4ff72 2024-12-10T20:16:06,315 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/73fe05a11c124c95a493448b04b4ff72, entries=150, sequenceid=81, filesize=11.7 K 2024-12-10T20:16:06,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/8b5ec2253e34439a8bbf8d05cdff2e41 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/8b5ec2253e34439a8bbf8d05cdff2e41 2024-12-10T20:16:06,344 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/8b5ec2253e34439a8bbf8d05cdff2e41, entries=150, sequenceid=81, filesize=11.7 K 2024-12-10T20:16:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c8a67a494bbd49eb8093ac3b3bbb3d02 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c8a67a494bbd49eb8093ac3b3bbb3d02 2024-12-10T20:16:06,351 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c8a67a494bbd49eb8093ac3b3bbb3d02, entries=150, sequenceid=81, filesize=11.7 K 2024-12-10T20:16:06,352 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d574f3055e0a5aa98dce8c43f5c73a8d in 789ms, sequenceid=81, compaction requested=false 2024-12-10T20:16:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-10T20:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-10T20:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:06,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-10T20:16:06,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3540 sec 2024-12-10T20:16:06,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:16:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:06,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.3760 sec 2024-12-10T20:16:06,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4a75f65a66d346b8a31e162c863023f6 is 50, key is test_row_0/A:col10/1733861765686/Put/seqid=0 2024-12-10T20:16:06,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742080_1256 (size=14341) 2024-12-10T20:16:06,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861826510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861826513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,555 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/824fffaf6c5b481f82d0afb04cc14c6e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/824fffaf6c5b481f82d0afb04cc14c6e 2024-12-10T20:16:06,573 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 824fffaf6c5b481f82d0afb04cc14c6e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:06,573 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:06,573 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861765543; duration=0sec 2024-12-10T20:16:06,573 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:06,573 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861826629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861826629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861826837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861826846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4a75f65a66d346b8a31e162c863023f6 2024-12-10T20:16:06,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861826899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,904 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861826901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,908 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861826907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:06,913 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:06,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/85f6248f02534854b3134f44fad1bbdf is 50, key is test_row_0/B:col10/1733861765686/Put/seqid=0 2024-12-10T20:16:06,931 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:16:06,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742081_1257 (size=12001) 2024-12-10T20:16:06,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/85f6248f02534854b3134f44fad1bbdf 2024-12-10T20:16:07,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ac82f0475f404bd2a6e6ee432fa5199f is 50, key is test_row_0/C:col10/1733861765686/Put/seqid=0 2024-12-10T20:16:07,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742082_1258 (size=12001) 2024-12-10T20:16:07,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ac82f0475f404bd2a6e6ee432fa5199f 2024-12-10T20:16:07,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4a75f65a66d346b8a31e162c863023f6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a75f65a66d346b8a31e162c863023f6 2024-12-10T20:16:07,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a75f65a66d346b8a31e162c863023f6, entries=200, sequenceid=96, filesize=14.0 K 2024-12-10T20:16:07,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/85f6248f02534854b3134f44fad1bbdf as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/85f6248f02534854b3134f44fad1bbdf 2024-12-10T20:16:07,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/85f6248f02534854b3134f44fad1bbdf, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T20:16:07,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ac82f0475f404bd2a6e6ee432fa5199f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ac82f0475f404bd2a6e6ee432fa5199f 2024-12-10T20:16:07,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ac82f0475f404bd2a6e6ee432fa5199f, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T20:16:07,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d574f3055e0a5aa98dce8c43f5c73a8d in 761ms, sequenceid=96, compaction requested=true 2024-12-10T20:16:07,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:07,129 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:07,130 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:07,130 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:07,130 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:07,131 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/dc653801d6bc4322b051ede096c0d62f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/73fe05a11c124c95a493448b04b4ff72, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a75f65a66d346b8a31e162c863023f6] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.5 K 2024-12-10T20:16:07,131 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc653801d6bc4322b051ede096c0d62f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733861764722 2024-12-10T20:16:07,131 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fe05a11c124c95a493448b04b4ff72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733861765324 2024-12-10T20:16:07,132 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a75f65a66d346b8a31e162c863023f6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733861765686 2024-12-10T20:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:16:07,135 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:07,141 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:07,141 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:07,141 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:07,141 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6448ae8462f8403c82e6f4a0f7551ac4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/8b5ec2253e34439a8bbf8d05cdff2e41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/85f6248f02534854b3134f44fad1bbdf] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.3 K 2024-12-10T20:16:07,142 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6448ae8462f8403c82e6f4a0f7551ac4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733861764722 2024-12-10T20:16:07,142 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5ec2253e34439a8bbf8d05cdff2e41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733861765324 2024-12-10T20:16:07,143 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f6248f02534854b3134f44fad1bbdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733861765686 2024-12-10T20:16:07,151 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:07,151 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/7ae3d44ddd0a4c4b94a5414be0aade9a is 50, key is test_row_0/A:col10/1733861765686/Put/seqid=0 2024-12-10T20:16:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:07,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T20:16:07,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733861766432, max timestamp=1733861767175Num uniques 859; 2024-12-10T20:16:07,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:07,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733861766432, max timestamp=1733861767175Num uniques 859; 2024-12-10T20:16:07,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:07,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733861766432, max timestamp=1733861767175Num uniques 859; 2024-12-10T20:16:07,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:07,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:07,242 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#214 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:07,243 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/58a90aa9544244ee9b7f74b93d942ed6 is 50, key is test_row_0/B:col10/1733861765686/Put/seqid=0 2024-12-10T20:16:07,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742083_1259 (size=12207) 2024-12-10T20:16:07,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f392f6cad38a4faa9e0c9fb1d5180867 is 50, key is test_row_0/A:col10/1733861766432/Put/seqid=0 2024-12-10T20:16:07,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742084_1260 (size=12207) 2024-12-10T20:16:07,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742085_1261 (size=12001) 2024-12-10T20:16:07,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f392f6cad38a4faa9e0c9fb1d5180867 2024-12-10T20:16:07,354 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/58a90aa9544244ee9b7f74b93d942ed6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/58a90aa9544244ee9b7f74b93d942ed6 2024-12-10T20:16:07,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/509542d984bc41aea1eb56489e5ed139 is 50, key is test_row_0/B:col10/1733861766432/Put/seqid=0 2024-12-10T20:16:07,367 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 58a90aa9544244ee9b7f74b93d942ed6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:07,367 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:07,367 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861767133; duration=0sec 2024-12-10T20:16:07,367 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:07,367 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:07,367 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:07,372 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:07,372 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:07,372 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:07,373 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/824fffaf6c5b481f82d0afb04cc14c6e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c8a67a494bbd49eb8093ac3b3bbb3d02, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ac82f0475f404bd2a6e6ee432fa5199f] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.3 K 2024-12-10T20:16:07,373 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 824fffaf6c5b481f82d0afb04cc14c6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733861764722 2024-12-10T20:16:07,374 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c8a67a494bbd49eb8093ac3b3bbb3d02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733861765324 2024-12-10T20:16:07,374 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ac82f0475f404bd2a6e6ee432fa5199f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733861765686 2024-12-10T20:16:07,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861767308, max timestamp=1733861767389Num uniques 859; 2024-12-10T20:16:07,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861767308, max timestamp=1733861767389Num uniques 859; 2024-12-10T20:16:07,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:07,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:07,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861767308, max timestamp=1733861767389Num uniques 859; 2024-12-10T20:16:07,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:07,419 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:07,420 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/7d22c2c7da874691ad725b9476eced59 is 50, key is test_row_0/C:col10/1733861765686/Put/seqid=0 2024-12-10T20:16:07,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742086_1262 (size=12001) 2024-12-10T20:16:07,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/509542d984bc41aea1eb56489e5ed139 2024-12-10T20:16:07,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/4d4f1587a23240d48394be6a58774681 is 50, key is test_row_0/C:col10/1733861766432/Put/seqid=0 2024-12-10T20:16:07,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742087_1263 (size=12207) 2024-12-10T20:16:07,481 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/7d22c2c7da874691ad725b9476eced59 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7d22c2c7da874691ad725b9476eced59 2024-12-10T20:16:07,486 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 7d22c2c7da874691ad725b9476eced59(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:07,487 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:07,487 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861767133; duration=0sec 2024-12-10T20:16:07,487 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:07,487 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:07,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742088_1264 (size=12001) 2024-12-10T20:16:07,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/4d4f1587a23240d48394be6a58774681 2024-12-10T20:16:07,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f392f6cad38a4faa9e0c9fb1d5180867 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f392f6cad38a4faa9e0c9fb1d5180867 2024-12-10T20:16:07,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f392f6cad38a4faa9e0c9fb1d5180867, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T20:16:07,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/509542d984bc41aea1eb56489e5ed139 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/509542d984bc41aea1eb56489e5ed139 2024-12-10T20:16:07,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/509542d984bc41aea1eb56489e5ed139, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T20:16:07,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/4d4f1587a23240d48394be6a58774681 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/4d4f1587a23240d48394be6a58774681 2024-12-10T20:16:07,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/4d4f1587a23240d48394be6a58774681, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T20:16:07,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=114.05 KB/116790 for d574f3055e0a5aa98dce8c43f5c73a8d in 351ms, sequenceid=118, compaction requested=false 2024-12-10T20:16:07,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:07,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-10T20:16:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:07,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/b623132421a0429fa5f029421b83caf9 is 50, key is test_row_0/A:col10/1733861767551/Put/seqid=0 2024-12-10T20:16:07,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:07,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861827661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861827657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742089_1265 (size=14541) 2024-12-10T20:16:07,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/b623132421a0429fa5f029421b83caf9 2024-12-10T20:16:07,701 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/7ae3d44ddd0a4c4b94a5414be0aade9a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7ae3d44ddd0a4c4b94a5414be0aade9a 2024-12-10T20:16:07,711 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 7ae3d44ddd0a4c4b94a5414be0aade9a(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:07,711 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:07,711 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861767129; duration=0sec 2024-12-10T20:16:07,711 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:07,711 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:07,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/0c7099991e0543c1b6e9a80424169323 is 50, key is test_row_0/B:col10/1733861767551/Put/seqid=0 2024-12-10T20:16:07,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742090_1266 (size=12151) 2024-12-10T20:16:07,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/0c7099991e0543c1b6e9a80424169323 2024-12-10T20:16:07,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6c502eddae8545e2b0ae507110e005ff is 50, key is test_row_0/C:col10/1733861767551/Put/seqid=0 2024-12-10T20:16:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861827777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861827777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:07,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742091_1267 (size=12151) 2024-12-10T20:16:07,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6c502eddae8545e2b0ae507110e005ff 2024-12-10T20:16:07,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/b623132421a0429fa5f029421b83caf9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b623132421a0429fa5f029421b83caf9 2024-12-10T20:16:07,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b623132421a0429fa5f029421b83caf9, entries=200, sequenceid=155, filesize=14.2 K 2024-12-10T20:16:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/0c7099991e0543c1b6e9a80424169323 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0c7099991e0543c1b6e9a80424169323 2024-12-10T20:16:07,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0c7099991e0543c1b6e9a80424169323, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T20:16:07,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6c502eddae8545e2b0ae507110e005ff as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6c502eddae8545e2b0ae507110e005ff 2024-12-10T20:16:07,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6c502eddae8545e2b0ae507110e005ff, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T20:16:07,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for d574f3055e0a5aa98dce8c43f5c73a8d in 286ms, sequenceid=155, compaction requested=true 2024-12-10T20:16:07,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:07,867 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:07,867 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:07,868 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:07,868 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:07,868 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:07,868 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7ae3d44ddd0a4c4b94a5414be0aade9a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f392f6cad38a4faa9e0c9fb1d5180867, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b623132421a0429fa5f029421b83caf9] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.8 K 2024-12-10T20:16:07,870 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae3d44ddd0a4c4b94a5414be0aade9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733861765686 2024-12-10T20:16:07,871 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:07,871 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f392f6cad38a4faa9e0c9fb1d5180867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733861766432 2024-12-10T20:16:07,871 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:07,871 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:07,871 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/58a90aa9544244ee9b7f74b93d942ed6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/509542d984bc41aea1eb56489e5ed139, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0c7099991e0543c1b6e9a80424169323] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.5 K 2024-12-10T20:16:07,872 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b623132421a0429fa5f029421b83caf9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733861767460 2024-12-10T20:16:07,872 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a90aa9544244ee9b7f74b93d942ed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733861765686 2024-12-10T20:16:07,878 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 509542d984bc41aea1eb56489e5ed139, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733861766432 2024-12-10T20:16:07,879 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7099991e0543c1b6e9a80424169323, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733861767507 2024-12-10T20:16:07,916 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:07,917 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f1938b81c6ab47e8b34a82e3c8d4aa5f is 50, key is test_row_0/A:col10/1733861767551/Put/seqid=0 2024-12-10T20:16:07,925 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:07,926 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/1394e4f572d34651a0319680fe3a5cc5 is 50, key is test_row_0/B:col10/1733861767551/Put/seqid=0 2024-12-10T20:16:07,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:16:07,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:07,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:07,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:07,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:07,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:07,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:08,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742092_1268 (size=12459) 2024-12-10T20:16:08,023 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/f1938b81c6ab47e8b34a82e3c8d4aa5f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f1938b81c6ab47e8b34a82e3c8d4aa5f 2024-12-10T20:16:08,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742093_1269 (size=12459) 2024-12-10T20:16:08,035 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into f1938b81c6ab47e8b34a82e3c8d4aa5f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:08,035 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:08,035 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861767866; duration=0sec 2024-12-10T20:16:08,035 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:08,035 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:08,036 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:08,039 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:08,039 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:08,044 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/1394e4f572d34651a0319680fe3a5cc5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1394e4f572d34651a0319680fe3a5cc5 2024-12-10T20:16:08,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3ee3dc5de0a041cfb52b6f033353ce9b is 50, key is test_row_0/A:col10/1733861767991/Put/seqid=0 2024-12-10T20:16:08,060 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,061 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7d22c2c7da874691ad725b9476eced59, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/4d4f1587a23240d48394be6a58774681, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6c502eddae8545e2b0ae507110e005ff] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.5 K 2024-12-10T20:16:08,061 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d22c2c7da874691ad725b9476eced59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733861765686 2024-12-10T20:16:08,063 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4f1587a23240d48394be6a58774681, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733861766432 2024-12-10T20:16:08,064 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c502eddae8545e2b0ae507110e005ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733861767507 2024-12-10T20:16:08,065 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 1394e4f572d34651a0319680fe3a5cc5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:08,065 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:08,065 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861767867; duration=0sec 2024-12-10T20:16:08,065 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:08,065 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:08,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861828103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861828111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,123 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#225 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:08,123 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/fdeabeabd30a4397ae369b4411e44e2f is 50, key is test_row_0/C:col10/1733861767551/Put/seqid=0 2024-12-10T20:16:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742094_1270 (size=14541) 2024-12-10T20:16:08,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3ee3dc5de0a041cfb52b6f033353ce9b 2024-12-10T20:16:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T20:16:08,176 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:08,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-10T20:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T20:16:08,189 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:08,190 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:08,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:08,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742095_1271 (size=12459) 2024-12-10T20:16:08,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c6936b9add6f41b398e37a1e539e94ab is 50, key is test_row_0/B:col10/1733861767991/Put/seqid=0 2024-12-10T20:16:08,216 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/fdeabeabd30a4397ae369b4411e44e2f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/fdeabeabd30a4397ae369b4411e44e2f 2024-12-10T20:16:08,231 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into fdeabeabd30a4397ae369b4411e44e2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:08,231 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:08,231 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861767867; duration=0sec 2024-12-10T20:16:08,231 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:08,231 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861828228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861828229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742096_1272 (size=12151) 2024-12-10T20:16:08,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c6936b9add6f41b398e37a1e539e94ab 2024-12-10T20:16:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T20:16:08,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/708b60f7e85e465894b1a108c0abe031 is 50, key is test_row_0/C:col10/1733861767991/Put/seqid=0 2024-12-10T20:16:08,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742097_1273 (size=12151) 2024-12-10T20:16:08,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/708b60f7e85e465894b1a108c0abe031 2024-12-10T20:16:08,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T20:16:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3ee3dc5de0a041cfb52b6f033353ce9b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3ee3dc5de0a041cfb52b6f033353ce9b 2024-12-10T20:16:08,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3ee3dc5de0a041cfb52b6f033353ce9b, entries=200, sequenceid=169, filesize=14.2 K 2024-12-10T20:16:08,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c6936b9add6f41b398e37a1e539e94ab as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c6936b9add6f41b398e37a1e539e94ab 2024-12-10T20:16:08,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c6936b9add6f41b398e37a1e539e94ab, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T20:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/708b60f7e85e465894b1a108c0abe031 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/708b60f7e85e465894b1a108c0abe031 2024-12-10T20:16:08,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/708b60f7e85e465894b1a108c0abe031, entries=150, sequenceid=169, filesize=11.9 K 2024-12-10T20:16:08,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d574f3055e0a5aa98dce8c43f5c73a8d in 425ms, sequenceid=169, compaction requested=false 2024-12-10T20:16:08,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:08,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-10T20:16:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:08,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861768092, max timestamp=1733861768453Num uniques 859; 2024-12-10T20:16:08,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:08,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861768092, max timestamp=1733861768453Num uniques 859; 2024-12-10T20:16:08,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:08,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:08,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:08,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:08,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:08,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:08,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:08,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/57e7164e08394b9296689afcaf74ff64 is 50, key is test_row_0/A:col10/1733861768099/Put/seqid=0 2024-12-10T20:16:08,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T20:16:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742098_1274 (size=12151) 2024-12-10T20:16:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/57e7164e08394b9296689afcaf74ff64 2024-12-10T20:16:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T20:16:08,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ef2612579d134b7a958af47a9c3923d1 is 50, key is test_row_0/B:col10/1733861768099/Put/seqid=0 2024-12-10T20:16:08,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861768539, max timestamp=1733861768567Num uniques 859; 2024-12-10T20:16:08,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:08,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861768539, max timestamp=1733861768567Num uniques 859; 2024-12-10T20:16:08,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:08,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733861768539, max timestamp=1733861768567Num uniques 859; 2024-12-10T20:16:08,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:08,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742099_1275 (size=12151) 2024-12-10T20:16:08,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T20:16:08,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861828660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861828660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861828777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861828779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T20:16:08,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T20:16:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861828921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,926 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:08,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861828929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,934 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861828936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:08,941 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:08,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ef2612579d134b7a958af47a9c3923d1 2024-12-10T20:16:08,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:08,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T20:16:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:08,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861828987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861829001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c2c9909b3aa34a00a53316b760223fb0 is 50, key is test_row_0/C:col10/1733861768099/Put/seqid=0 2024-12-10T20:16:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742100_1276 (size=12151) 2024-12-10T20:16:09,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c2c9909b3aa34a00a53316b760223fb0 2024-12-10T20:16:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/57e7164e08394b9296689afcaf74ff64 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/57e7164e08394b9296689afcaf74ff64 2024-12-10T20:16:09,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/57e7164e08394b9296689afcaf74ff64, entries=150, sequenceid=193, filesize=11.9 K 2024-12-10T20:16:09,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ef2612579d134b7a958af47a9c3923d1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ef2612579d134b7a958af47a9c3923d1 2024-12-10T20:16:09,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ef2612579d134b7a958af47a9c3923d1, entries=150, sequenceid=193, filesize=11.9 K 2024-12-10T20:16:09,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c2c9909b3aa34a00a53316b760223fb0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c9909b3aa34a00a53316b760223fb0 2024-12-10T20:16:09,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c9909b3aa34a00a53316b760223fb0, entries=150, sequenceid=193, filesize=11.9 K 2024-12-10T20:16:09,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~169.20 KB/173264, currentSize=127.47 KB/130530 for d574f3055e0a5aa98dce8c43f5c73a8d in 662ms, sequenceid=193, compaction requested=true 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:09,118 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:09,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:16:09,120 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:09,140 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:09,140 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:09,140 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:09,140 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f1938b81c6ab47e8b34a82e3c8d4aa5f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3ee3dc5de0a041cfb52b6f033353ce9b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/57e7164e08394b9296689afcaf74ff64] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=38.2 K 2024-12-10T20:16:09,140 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:09,141 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:09,141 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:09,141 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1394e4f572d34651a0319680fe3a5cc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c6936b9add6f41b398e37a1e539e94ab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ef2612579d134b7a958af47a9c3923d1] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.9 K 2024-12-10T20:16:09,144 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f1938b81c6ab47e8b34a82e3c8d4aa5f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733861767507 2024-12-10T20:16:09,144 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1394e4f572d34651a0319680fe3a5cc5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733861767507 2024-12-10T20:16:09,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:09,149 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:09,152 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6936b9add6f41b398e37a1e539e94ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733861767606 2024-12-10T20:16:09,153 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee3dc5de0a041cfb52b6f033353ce9b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733861767606 2024-12-10T20:16:09,157 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e7164e08394b9296689afcaf74ff64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733861768099 2024-12-10T20:16:09,157 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2612579d134b7a958af47a9c3923d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733861768099 2024-12-10T20:16:09,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/9d45bc5b3b05494ba371580ab72c1c68 is 50, key is test_row_0/A:col10/1733861768658/Put/seqid=0 2024-12-10T20:16:09,207 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:09,209 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4f812cdeae0a400c9e877c05a64d71e1 is 50, key is test_row_0/A:col10/1733861768099/Put/seqid=0 2024-12-10T20:16:09,216 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:09,216 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/388d835521c04a63bf39f8df3ad64b2c is 50, key is test_row_0/B:col10/1733861768099/Put/seqid=0 2024-12-10T20:16:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742101_1277 (size=12151) 2024-12-10T20:16:09,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742103_1279 (size=12561) 2024-12-10T20:16:09,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742102_1278 (size=12561) 2024-12-10T20:16:09,294 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/388d835521c04a63bf39f8df3ad64b2c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/388d835521c04a63bf39f8df3ad64b2c 2024-12-10T20:16:09,302 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4f812cdeae0a400c9e877c05a64d71e1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4f812cdeae0a400c9e877c05a64d71e1 2024-12-10T20:16:09,314 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 4f812cdeae0a400c9e877c05a64d71e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:09,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:09,315 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:09,316 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861769118; duration=0sec 2024-12-10T20:16:09,316 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:09,316 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:09,316 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:09,323 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:09,323 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:09,323 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:09,323 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/fdeabeabd30a4397ae369b4411e44e2f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/708b60f7e85e465894b1a108c0abe031, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c9909b3aa34a00a53316b760223fb0] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=35.9 K 2024-12-10T20:16:09,324 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting fdeabeabd30a4397ae369b4411e44e2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733861767507 2024-12-10T20:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T20:16:09,328 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 708b60f7e85e465894b1a108c0abe031, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733861767606 2024-12-10T20:16:09,328 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c9909b3aa34a00a53316b760223fb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733861768099 2024-12-10T20:16:09,330 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 388d835521c04a63bf39f8df3ad64b2c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:09,330 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:09,330 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861769118; duration=0sec 2024-12-10T20:16:09,330 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:09,330 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:09,344 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:09,345 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/a791ba8933c04a36b6e7690211fe6d7a is 50, key is test_row_0/C:col10/1733861768099/Put/seqid=0 2024-12-10T20:16:09,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861829363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861829365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742104_1280 (size=12561) 2024-12-10T20:16:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861829477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861829478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,633 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/9d45bc5b3b05494ba371580ab72c1c68 2024-12-10T20:16:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c10cd159140f452ca84967429d44fa08 is 50, key is test_row_0/B:col10/1733861768658/Put/seqid=0 2024-12-10T20:16:09,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742105_1281 (size=12151) 2024-12-10T20:16:09,676 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c10cd159140f452ca84967429d44fa08 2024-12-10T20:16:09,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/7070f1b543074f0b9cf9663d90caf7b0 is 50, key is test_row_0/C:col10/1733861768658/Put/seqid=0 2024-12-10T20:16:09,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861829689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861829690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742106_1282 (size=12151) 2024-12-10T20:16:09,722 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/7070f1b543074f0b9cf9663d90caf7b0 2024-12-10T20:16:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/9d45bc5b3b05494ba371580ab72c1c68 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9d45bc5b3b05494ba371580ab72c1c68 2024-12-10T20:16:09,741 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9d45bc5b3b05494ba371580ab72c1c68, entries=150, sequenceid=228, filesize=11.9 K 2024-12-10T20:16:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c10cd159140f452ca84967429d44fa08 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c10cd159140f452ca84967429d44fa08 2024-12-10T20:16:09,747 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c10cd159140f452ca84967429d44fa08, entries=150, sequenceid=228, filesize=11.9 K 2024-12-10T20:16:09,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/7070f1b543074f0b9cf9663d90caf7b0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7070f1b543074f0b9cf9663d90caf7b0 2024-12-10T20:16:09,755 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7070f1b543074f0b9cf9663d90caf7b0, entries=150, sequenceid=228, filesize=11.9 K 2024-12-10T20:16:09,759 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d574f3055e0a5aa98dce8c43f5c73a8d in 610ms, sequenceid=228, compaction requested=false 2024-12-10T20:16:09,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:09,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:09,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-10T20:16:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-10T20:16:09,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-10T20:16:09,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-12-10T20:16:09,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5880 sec 2024-12-10T20:16:09,798 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/a791ba8933c04a36b6e7690211fe6d7a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/a791ba8933c04a36b6e7690211fe6d7a 2024-12-10T20:16:09,813 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into a791ba8933c04a36b6e7690211fe6d7a(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:09,813 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:09,813 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861769118; duration=0sec 2024-12-10T20:16:09,815 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:09,815 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:10,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T20:16:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:10,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:10,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/205c5bcb4f1e4eb590b1f2f4871a847a is 50, key is test_row_0/A:col10/1733861770011/Put/seqid=0 2024-12-10T20:16:10,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742107_1283 (size=14541) 2024-12-10T20:16:10,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861830126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861830137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861830237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861830261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T20:16:10,338 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:10,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-10T20:16:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T20:16:10,345 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:10,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:10,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T20:16:10,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861830447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861830469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/205c5bcb4f1e4eb590b1f2f4871a847a 2024-12-10T20:16:10,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T20:16:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:10,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/887f421f0409417badbe94b26002e628 is 50, key is test_row_0/B:col10/1733861770011/Put/seqid=0 2024-12-10T20:16:10,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742108_1284 (size=12151) 2024-12-10T20:16:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T20:16:10,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T20:16:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:10,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861830757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861830788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:10,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T20:16:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:10,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T20:16:10,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/887f421f0409417badbe94b26002e628 2024-12-10T20:16:10,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/5be250b3a9114d939db4f590ea948bce is 50, key is test_row_0/C:col10/1733861770011/Put/seqid=0 2024-12-10T20:16:11,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T20:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:11,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:11,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742109_1285 (size=12151) 2024-12-10T20:16:11,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/5be250b3a9114d939db4f590ea948bce 2024-12-10T20:16:11,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/205c5bcb4f1e4eb590b1f2f4871a847a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/205c5bcb4f1e4eb590b1f2f4871a847a 2024-12-10T20:16:11,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/205c5bcb4f1e4eb590b1f2f4871a847a, entries=200, sequenceid=244, filesize=14.2 K 2024-12-10T20:16:11,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/887f421f0409417badbe94b26002e628 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/887f421f0409417badbe94b26002e628 2024-12-10T20:16:11,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/887f421f0409417badbe94b26002e628, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T20:16:11,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/5be250b3a9114d939db4f590ea948bce as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/5be250b3a9114d939db4f590ea948bce 2024-12-10T20:16:11,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/5be250b3a9114d939db4f590ea948bce, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T20:16:11,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d574f3055e0a5aa98dce8c43f5c73a8d in 1053ms, sequenceid=244, compaction requested=true 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:11,070 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:16:11,071 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:11,071 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:11,071 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,072 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4f812cdeae0a400c9e877c05a64d71e1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9d45bc5b3b05494ba371580ab72c1c68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/205c5bcb4f1e4eb590b1f2f4871a847a] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=38.3 K 2024-12-10T20:16:11,072 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f812cdeae0a400c9e877c05a64d71e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733861768099 2024-12-10T20:16:11,072 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:11,073 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d45bc5b3b05494ba371580ab72c1c68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733861768646 2024-12-10T20:16:11,073 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 205c5bcb4f1e4eb590b1f2f4871a847a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861769340 2024-12-10T20:16:11,074 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:11,074 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:11,074 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,074 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/388d835521c04a63bf39f8df3ad64b2c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c10cd159140f452ca84967429d44fa08, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/887f421f0409417badbe94b26002e628] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.0 K 2024-12-10T20:16:11,075 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 388d835521c04a63bf39f8df3ad64b2c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733861768099 2024-12-10T20:16:11,076 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c10cd159140f452ca84967429d44fa08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733861768646 2024-12-10T20:16:11,076 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 887f421f0409417badbe94b26002e628, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861769340 2024-12-10T20:16:11,083 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:11,084 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/93e59093f42d4c67b3b09e9672a6ee36 is 50, key is test_row_0/A:col10/1733861770011/Put/seqid=0 2024-12-10T20:16:11,087 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:11,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ee8eeeab555b431b918eab695f36dd90 is 50, key is test_row_0/B:col10/1733861770011/Put/seqid=0 2024-12-10T20:16:11,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742110_1286 (size=12663) 2024-12-10T20:16:11,131 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/93e59093f42d4c67b3b09e9672a6ee36 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/93e59093f42d4c67b3b09e9672a6ee36 2024-12-10T20:16:11,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742111_1287 (size=12663) 2024-12-10T20:16:11,142 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 93e59093f42d4c67b3b09e9672a6ee36(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:11,142 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:11,142 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861771070; duration=0sec 2024-12-10T20:16:11,142 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:11,142 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:11,142 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:11,144 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:11,144 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:11,144 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,144 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/a791ba8933c04a36b6e7690211fe6d7a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7070f1b543074f0b9cf9663d90caf7b0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/5be250b3a9114d939db4f590ea948bce] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.0 K 2024-12-10T20:16:11,145 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a791ba8933c04a36b6e7690211fe6d7a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733861768099 2024-12-10T20:16:11,147 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7070f1b543074f0b9cf9663d90caf7b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733861768646 2024-12-10T20:16:11,147 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be250b3a9114d939db4f590ea948bce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861769340 2024-12-10T20:16:11,153 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ee8eeeab555b431b918eab695f36dd90 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ee8eeeab555b431b918eab695f36dd90 2024-12-10T20:16:11,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T20:16:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,161 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:16:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:11,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:11,171 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into ee8eeeab555b431b918eab695f36dd90(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:11,171 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:11,171 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861771070; duration=0sec 2024-12-10T20:16:11,171 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:11,171 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:11,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/29723507b50d4f45bc37df60c5f14b7e is 50, key is test_row_0/A:col10/1733861770128/Put/seqid=0 2024-12-10T20:16:11,181 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:11,181 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/be2633b28d52400abf488cd6985f0c9b is 50, key is test_row_0/C:col10/1733861770011/Put/seqid=0 2024-12-10T20:16:11,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742113_1289 (size=12663) 2024-12-10T20:16:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742112_1288 (size=12301) 2024-12-10T20:16:11,225 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/29723507b50d4f45bc37df60c5f14b7e 2024-12-10T20:16:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/47177f97be5a4f56b19d26bc81b0883c is 50, key is test_row_0/B:col10/1733861770128/Put/seqid=0 2024-12-10T20:16:11,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742114_1290 (size=12301) 2024-12-10T20:16:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:11,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861831344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:11,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:11,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861831344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:11,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861831457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861831457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T20:16:11,633 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/be2633b28d52400abf488cd6985f0c9b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/be2633b28d52400abf488cd6985f0c9b 2024-12-10T20:16:11,641 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into be2633b28d52400abf488cd6985f0c9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:11,641 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:11,641 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861771070; duration=0sec 2024-12-10T20:16:11,641 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:11,641 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:11,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861831667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861831669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:11,672 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/47177f97be5a4f56b19d26bc81b0883c 2024-12-10T20:16:11,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/f6e2c03699034a319b43916692697017 is 50, key is test_row_0/C:col10/1733861770128/Put/seqid=0 2024-12-10T20:16:11,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742115_1291 (size=12301) 2024-12-10T20:16:11,726 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/f6e2c03699034a319b43916692697017 2024-12-10T20:16:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/29723507b50d4f45bc37df60c5f14b7e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/29723507b50d4f45bc37df60c5f14b7e 2024-12-10T20:16:11,740 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/29723507b50d4f45bc37df60c5f14b7e, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T20:16:11,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/47177f97be5a4f56b19d26bc81b0883c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/47177f97be5a4f56b19d26bc81b0883c 2024-12-10T20:16:11,747 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/47177f97be5a4f56b19d26bc81b0883c, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T20:16:11,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/f6e2c03699034a319b43916692697017 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/f6e2c03699034a319b43916692697017 2024-12-10T20:16:11,754 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/f6e2c03699034a319b43916692697017, entries=150, sequenceid=264, filesize=12.0 K 2024-12-10T20:16:11,755 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for d574f3055e0a5aa98dce8c43f5c73a8d in 594ms, sequenceid=264, compaction requested=false 2024-12-10T20:16:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-10T20:16:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-10T20:16:11,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-10T20:16:11,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-12-10T20:16:11,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4240 sec 2024-12-10T20:16:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:11,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:16:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:11,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:11,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/1b379e486d03413db1424490cdcfeb72 is 50, key is test_row_0/A:col10/1733861771978/Put/seqid=0 2024-12-10T20:16:12,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742116_1292 (size=12301) 2024-12-10T20:16:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861832021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861832023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861832128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861832137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861832338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861832347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/1b379e486d03413db1424490cdcfeb72 2024-12-10T20:16:12,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/644c2f63791c44ddb826985700de6437 is 50, key is test_row_0/B:col10/1733861771978/Put/seqid=0 2024-12-10T20:16:12,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742117_1293 (size=12301) 2024-12-10T20:16:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T20:16:12,513 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:12,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-10T20:16:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T20:16:12,518 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:12,519 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:12,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T20:16:12,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861832657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T20:16:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:12,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:12,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861832668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T20:16:12,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T20:16:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:12,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:12,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/644c2f63791c44ddb826985700de6437 2024-12-10T20:16:12,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/875efa66327641faab9290d44ec5b9b7 is 50, key is test_row_0/C:col10/1733861771978/Put/seqid=0 2024-12-10T20:16:12,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742118_1294 (size=12301) 2024-12-10T20:16:12,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/875efa66327641faab9290d44ec5b9b7 2024-12-10T20:16:12,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/1b379e486d03413db1424490cdcfeb72 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b379e486d03413db1424490cdcfeb72 2024-12-10T20:16:12,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b379e486d03413db1424490cdcfeb72, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T20:16:12,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861832956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,966 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:12,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/644c2f63791c44ddb826985700de6437 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/644c2f63791c44ddb826985700de6437 2024-12-10T20:16:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/644c2f63791c44ddb826985700de6437, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T20:16:12,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/875efa66327641faab9290d44ec5b9b7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/875efa66327641faab9290d44ec5b9b7 2024-12-10T20:16:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861832967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,985 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861832981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:12,991 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:12,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/875efa66327641faab9290d44ec5b9b7, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T20:16:12,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d574f3055e0a5aa98dce8c43f5c73a8d in 1018ms, sequenceid=281, compaction requested=true 2024-12-10T20:16:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:12,998 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:12,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:12,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,005 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:13,005 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:13,005 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:13,005 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:13,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:13,005 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/93e59093f42d4c67b3b09e9672a6ee36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/29723507b50d4f45bc37df60c5f14b7e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b379e486d03413db1424490cdcfeb72] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.4 K 2024-12-10T20:16:13,009 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:13,009 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:13,010 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e59093f42d4c67b3b09e9672a6ee36, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861769340 2024-12-10T20:16:13,012 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 29723507b50d4f45bc37df60c5f14b7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733861770100 2024-12-10T20:16:13,012 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,013 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ee8eeeab555b431b918eab695f36dd90, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/47177f97be5a4f56b19d26bc81b0883c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/644c2f63791c44ddb826985700de6437] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.4 K 2024-12-10T20:16:13,014 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b379e486d03413db1424490cdcfeb72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733861771284 2024-12-10T20:16:13,017 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8eeeab555b431b918eab695f36dd90, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861769340 2024-12-10T20:16:13,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/ea64870ca2644a1393855d9d8edadf67 is 50, key is test_row_0/A:col10/1733861772018/Put/seqid=0 2024-12-10T20:16:13,024 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47177f97be5a4f56b19d26bc81b0883c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733861770100 2024-12-10T20:16:13,024 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644c2f63791c44ddb826985700de6437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733861771284 2024-12-10T20:16:13,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742119_1295 (size=12301) 2024-12-10T20:16:13,061 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/ea64870ca2644a1393855d9d8edadf67 2024-12-10T20:16:13,066 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:13,067 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/23c0b6ed7cfa4ec88acb8c243d82ffd4 is 50, key is test_row_0/A:col10/1733861771978/Put/seqid=0 2024-12-10T20:16:13,070 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:13,071 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/32a93d09e8e442409477c39a0e5439d4 is 50, key is test_row_0/B:col10/1733861771978/Put/seqid=0 2024-12-10T20:16:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/4f77097a62434108a8e232bb9e338382 is 50, key is test_row_0/B:col10/1733861772018/Put/seqid=0 2024-12-10T20:16:13,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742120_1296 (size=12915) 2024-12-10T20:16:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T20:16:13,151 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/23c0b6ed7cfa4ec88acb8c243d82ffd4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/23c0b6ed7cfa4ec88acb8c243d82ffd4 2024-12-10T20:16:13,162 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 23c0b6ed7cfa4ec88acb8c243d82ffd4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:13,162 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:13,162 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861772998; duration=0sec 2024-12-10T20:16:13,162 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:13,162 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:13,162 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:13,164 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:13,164 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:13,164 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,164 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/be2633b28d52400abf488cd6985f0c9b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/f6e2c03699034a319b43916692697017, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/875efa66327641faab9290d44ec5b9b7] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.4 K 2024-12-10T20:16:13,166 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting be2633b28d52400abf488cd6985f0c9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733861769340 2024-12-10T20:16:13,167 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e2c03699034a319b43916692697017, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733861770100 2024-12-10T20:16:13,168 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 875efa66327641faab9290d44ec5b9b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733861771284 2024-12-10T20:16:13,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:13,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742122_1298 (size=12301) 2024-12-10T20:16:13,216 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/4f77097a62434108a8e232bb9e338382 2024-12-10T20:16:13,230 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:13,230 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/60379e909f584941b9c21bb57786bcde is 50, key is test_row_0/C:col10/1733861771978/Put/seqid=0 2024-12-10T20:16:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/38b541d568f44f5ba6d9fc7907cdc792 is 50, key is test_row_0/C:col10/1733861772018/Put/seqid=0 2024-12-10T20:16:13,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861833254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861833257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742121_1297 (size=12915) 2024-12-10T20:16:13,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742123_1299 (size=12915) 2024-12-10T20:16:13,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742124_1300 (size=12301) 2024-12-10T20:16:13,286 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/38b541d568f44f5ba6d9fc7907cdc792 2024-12-10T20:16:13,287 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/32a93d09e8e442409477c39a0e5439d4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/32a93d09e8e442409477c39a0e5439d4 2024-12-10T20:16:13,309 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/60379e909f584941b9c21bb57786bcde as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/60379e909f584941b9c21bb57786bcde 2024-12-10T20:16:13,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/ea64870ca2644a1393855d9d8edadf67 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ea64870ca2644a1393855d9d8edadf67 2024-12-10T20:16:13,318 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 32a93d09e8e442409477c39a0e5439d4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:13,318 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:13,320 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861772998; duration=0sec 2024-12-10T20:16:13,321 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:13,321 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:13,333 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 60379e909f584941b9c21bb57786bcde(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:13,333 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:13,333 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861772998; duration=0sec 2024-12-10T20:16:13,335 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ea64870ca2644a1393855d9d8edadf67, entries=150, sequenceid=298, filesize=12.0 K 2024-12-10T20:16:13,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/4f77097a62434108a8e232bb9e338382 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/4f77097a62434108a8e232bb9e338382 2024-12-10T20:16:13,338 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:13,338 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:13,350 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/4f77097a62434108a8e232bb9e338382, entries=150, sequenceid=298, filesize=12.0 K 2024-12-10T20:16:13,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/38b541d568f44f5ba6d9fc7907cdc792 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38b541d568f44f5ba6d9fc7907cdc792 2024-12-10T20:16:13,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861833381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,390 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38b541d568f44f5ba6d9fc7907cdc792, entries=150, sequenceid=298, filesize=12.0 K 2024-12-10T20:16:13,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861833377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,394 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d574f3055e0a5aa98dce8c43f5c73a8d in 389ms, sequenceid=298, compaction requested=false 2024-12-10T20:16:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-10T20:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-10T20:16:13,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-10T20:16:13,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 879 msec 2024-12-10T20:16:13,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 892 msec 2024-12-10T20:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:13,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:16:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:13,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4b4569c458fa4a9b9dd594a24846dffc is 50, key is test_row_0/A:col10/1733861773247/Put/seqid=0 2024-12-10T20:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T20:16:13,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:13,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-10T20:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:13,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:13,661 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:13,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:13,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742125_1301 (size=12301) 2024-12-10T20:16:13,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4b4569c458fa4a9b9dd594a24846dffc 2024-12-10T20:16:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861833685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861833692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/5a7cf764bb294280899c56232fbbe3f6 is 50, key is test_row_0/B:col10/1733861773247/Put/seqid=0 2024-12-10T20:16:13,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742126_1302 (size=12301) 2024-12-10T20:16:13,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/5a7cf764bb294280899c56232fbbe3f6 2024-12-10T20:16:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:13,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/38f8a6e708c647bbbef03f9079e62e1b is 50, key is test_row_0/C:col10/1733861773247/Put/seqid=0 2024-12-10T20:16:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861833799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861833807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:13,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:13,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:13,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:13,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:13,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742127_1303 (size=12301) 2024-12-10T20:16:13,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:13,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861834009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861834019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/38f8a6e708c647bbbef03f9079e62e1b 2024-12-10T20:16:14,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4b4569c458fa4a9b9dd594a24846dffc as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4b4569c458fa4a9b9dd594a24846dffc 2024-12-10T20:16:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:14,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4b4569c458fa4a9b9dd594a24846dffc, entries=150, sequenceid=318, filesize=12.0 K 2024-12-10T20:16:14,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/5a7cf764bb294280899c56232fbbe3f6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/5a7cf764bb294280899c56232fbbe3f6 2024-12-10T20:16:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/5a7cf764bb294280899c56232fbbe3f6, entries=150, sequenceid=318, filesize=12.0 K 2024-12-10T20:16:14,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/38f8a6e708c647bbbef03f9079e62e1b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38f8a6e708c647bbbef03f9079e62e1b 2024-12-10T20:16:14,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38f8a6e708c647bbbef03f9079e62e1b, entries=150, sequenceid=318, filesize=12.0 K 2024-12-10T20:16:14,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d574f3055e0a5aa98dce8c43f5c73a8d in 724ms, sequenceid=318, compaction requested=true 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T20:16:14,330 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:14,330 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:14,331 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:14,331 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:14,331 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:14,331 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:14,331 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,331 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,331 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/23c0b6ed7cfa4ec88acb8c243d82ffd4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ea64870ca2644a1393855d9d8edadf67, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4b4569c458fa4a9b9dd594a24846dffc] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.6 K 2024-12-10T20:16:14,331 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/60379e909f584941b9c21bb57786bcde, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38b541d568f44f5ba6d9fc7907cdc792, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38f8a6e708c647bbbef03f9079e62e1b] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.6 K 2024-12-10T20:16:14,332 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 60379e909f584941b9c21bb57786bcde, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733861771284 2024-12-10T20:16:14,332 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b541d568f44f5ba6d9fc7907cdc792, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733861772008 2024-12-10T20:16:14,332 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f8a6e708c647bbbef03f9079e62e1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733861773247 2024-12-10T20:16:14,333 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c0b6ed7cfa4ec88acb8c243d82ffd4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733861771284 2024-12-10T20:16:14,333 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea64870ca2644a1393855d9d8edadf67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733861772008 2024-12-10T20:16:14,334 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b4569c458fa4a9b9dd594a24846dffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733861773247 2024-12-10T20:16:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:14,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T20:16:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:14,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4a19bafd5d504cd39243cc1d943c351c is 50, key is test_row_0/A:col10/1733861773684/Put/seqid=0 2024-12-10T20:16:14,363 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:14,363 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e39c24e646a84aa8985b3d29e1eee7e2 is 50, key is test_row_0/C:col10/1733861773247/Put/seqid=0 2024-12-10T20:16:14,371 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:14,372 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/1b7badbebcfa439db6aea8d46f4eba56 is 50, key is test_row_0/A:col10/1733861773247/Put/seqid=0 2024-12-10T20:16:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861834402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861834413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742129_1305 (size=13017) 2024-12-10T20:16:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742128_1304 (size=14741) 2024-12-10T20:16:14,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4a19bafd5d504cd39243cc1d943c351c 2024-12-10T20:16:14,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742130_1306 (size=13017) 2024-12-10T20:16:14,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d1ed7aa5d3d1415fad03f4ba82c2d73b is 50, key is test_row_0/B:col10/1733861773684/Put/seqid=0 2024-12-10T20:16:14,480 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/1b7badbebcfa439db6aea8d46f4eba56 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b7badbebcfa439db6aea8d46f4eba56 2024-12-10T20:16:14,497 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 1b7badbebcfa439db6aea8d46f4eba56(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:14,497 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:14,497 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861774330; duration=0sec 2024-12-10T20:16:14,497 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:14,498 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:14,498 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:14,499 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:14,499 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:14,499 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,499 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/32a93d09e8e442409477c39a0e5439d4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/4f77097a62434108a8e232bb9e338382, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/5a7cf764bb294280899c56232fbbe3f6] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.6 K 2024-12-10T20:16:14,499 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a93d09e8e442409477c39a0e5439d4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733861771284 2024-12-10T20:16:14,500 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f77097a62434108a8e232bb9e338382, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733861772008 2024-12-10T20:16:14,501 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a7cf764bb294280899c56232fbbe3f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733861773247 2024-12-10T20:16:14,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742131_1307 (size=12301) 2024-12-10T20:16:14,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d1ed7aa5d3d1415fad03f4ba82c2d73b 2024-12-10T20:16:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861834518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,525 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:14,526 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/a46c8fcb02a04eec935e95b1920f64e8 is 50, key is test_row_0/B:col10/1733861773247/Put/seqid=0 2024-12-10T20:16:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861834532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/576bed617d7e4ea2a6dd78803e867b51 is 50, key is test_row_0/C:col10/1733861773684/Put/seqid=0 2024-12-10T20:16:14,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742132_1308 (size=13017) 2024-12-10T20:16:14,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742133_1309 (size=12301) 2024-12-10T20:16:14,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/576bed617d7e4ea2a6dd78803e867b51 2024-12-10T20:16:14,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/4a19bafd5d504cd39243cc1d943c351c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a19bafd5d504cd39243cc1d943c351c 2024-12-10T20:16:14,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a19bafd5d504cd39243cc1d943c351c, entries=200, sequenceid=334, filesize=14.4 K 2024-12-10T20:16:14,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d1ed7aa5d3d1415fad03f4ba82c2d73b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1ed7aa5d3d1415fad03f4ba82c2d73b 2024-12-10T20:16:14,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1ed7aa5d3d1415fad03f4ba82c2d73b, entries=150, sequenceid=334, filesize=12.0 K 2024-12-10T20:16:14,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:14,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:14,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/576bed617d7e4ea2a6dd78803e867b51 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/576bed617d7e4ea2a6dd78803e867b51 2024-12-10T20:16:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/576bed617d7e4ea2a6dd78803e867b51, entries=150, sequenceid=334, filesize=12.0 K 2024-12-10T20:16:14,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d574f3055e0a5aa98dce8c43f5c73a8d in 281ms, sequenceid=334, compaction requested=false 2024-12-10T20:16:14,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:14,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:16:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:14,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/d22ceefab8114bfdbecbcddaf3ae3ff2 is 50, key is test_row_0/A:col10/1733861774731/Put/seqid=0 2024-12-10T20:16:14,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:14,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742134_1310 (size=14741) 2024-12-10T20:16:14,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861834820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861834828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,845 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e39c24e646a84aa8985b3d29e1eee7e2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e39c24e646a84aa8985b3d29e1eee7e2 2024-12-10T20:16:14,868 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into e39c24e646a84aa8985b3d29e1eee7e2(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:14,868 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:14,868 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861774330; duration=0sec 2024-12-10T20:16:14,868 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:14,868 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:14,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:14,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861834937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861834947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:14,979 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/a46c8fcb02a04eec935e95b1920f64e8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a46c8fcb02a04eec935e95b1920f64e8 2024-12-10T20:16:14,986 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into a46c8fcb02a04eec935e95b1920f64e8(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:14,986 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:14,986 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861774330; duration=0sec 2024-12-10T20:16:14,986 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:14,986 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:15,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861835156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:15,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861835157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:15,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/d22ceefab8114bfdbecbcddaf3ae3ff2 2024-12-10T20:16:15,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/a955783c2dea414a9007a3dd41f0adbb is 50, key is test_row_0/B:col10/1733861774731/Put/seqid=0 2024-12-10T20:16:15,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:15,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742135_1311 (size=12301) 2024-12-10T20:16:15,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861835476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861835477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:15,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:15,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/a955783c2dea414a9007a3dd41f0adbb 2024-12-10T20:16:15,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ea792f3a3eab4371aac1f96f08bea4d7 is 50, key is test_row_0/C:col10/1733861774731/Put/seqid=0 2024-12-10T20:16:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:15,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742136_1312 (size=12301) 2024-12-10T20:16:15,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ea792f3a3eab4371aac1f96f08bea4d7 2024-12-10T20:16:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/d22ceefab8114bfdbecbcddaf3ae3ff2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/d22ceefab8114bfdbecbcddaf3ae3ff2 2024-12-10T20:16:15,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:15,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:15,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/d22ceefab8114bfdbecbcddaf3ae3ff2, entries=200, sequenceid=352, filesize=14.4 K 2024-12-10T20:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/a955783c2dea414a9007a3dd41f0adbb as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a955783c2dea414a9007a3dd41f0adbb 2024-12-10T20:16:15,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a955783c2dea414a9007a3dd41f0adbb, entries=150, sequenceid=352, filesize=12.0 K 2024-12-10T20:16:15,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/ea792f3a3eab4371aac1f96f08bea4d7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ea792f3a3eab4371aac1f96f08bea4d7 2024-12-10T20:16:15,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ea792f3a3eab4371aac1f96f08bea4d7, entries=150, sequenceid=352, filesize=12.0 K 2024-12-10T20:16:15,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d574f3055e0a5aa98dce8c43f5c73a8d in 1209ms, sequenceid=352, compaction requested=true 2024-12-10T20:16:15,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:15,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:15,943 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:15,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:15,943 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:15,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:15,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:15,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:15,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:15,944 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:15,945 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:15,945 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,945 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a46c8fcb02a04eec935e95b1920f64e8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1ed7aa5d3d1415fad03f4ba82c2d73b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a955783c2dea414a9007a3dd41f0adbb] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.7 K 2024-12-10T20:16:15,945 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:15,945 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:15,945 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:15,945 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b7badbebcfa439db6aea8d46f4eba56, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a19bafd5d504cd39243cc1d943c351c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/d22ceefab8114bfdbecbcddaf3ae3ff2] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=41.5 K 2024-12-10T20:16:15,946 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a46c8fcb02a04eec935e95b1920f64e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733861773247 2024-12-10T20:16:15,946 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b7badbebcfa439db6aea8d46f4eba56, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733861773247 2024-12-10T20:16:15,947 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ed7aa5d3d1415fad03f4ba82c2d73b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733861773670 2024-12-10T20:16:15,947 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a19bafd5d504cd39243cc1d943c351c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733861773670 2024-12-10T20:16:15,951 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting d22ceefab8114bfdbecbcddaf3ae3ff2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733861774375 2024-12-10T20:16:15,951 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a955783c2dea414a9007a3dd41f0adbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733861774375 2024-12-10T20:16:15,967 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:15,967 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/269d374a2ca345b386ea248be3aba5b9 is 50, key is test_row_0/A:col10/1733861774731/Put/seqid=0 2024-12-10T20:16:15,974 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:15,974 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/2258e28340dd445eb74b78b93d5bd480 is 50, key is test_row_0/B:col10/1733861774731/Put/seqid=0 2024-12-10T20:16:15,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742137_1313 (size=13119) 2024-12-10T20:16:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:15,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T20:16:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:16,002 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/269d374a2ca345b386ea248be3aba5b9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/269d374a2ca345b386ea248be3aba5b9 2024-12-10T20:16:16,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742138_1314 (size=13119) 2024-12-10T20:16:16,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/74ff9ad9b1da4b7794617dc27746033b is 50, key is test_row_0/A:col10/1733861774806/Put/seqid=0 2024-12-10T20:16:16,032 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 269d374a2ca345b386ea248be3aba5b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:16,032 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:16,032 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861775942; duration=0sec 2024-12-10T20:16:16,033 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:16,033 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:16,033 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:16,037 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:16,037 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:16,037 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,037 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e39c24e646a84aa8985b3d29e1eee7e2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/576bed617d7e4ea2a6dd78803e867b51, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ea792f3a3eab4371aac1f96f08bea4d7] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.7 K 2024-12-10T20:16:16,037 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e39c24e646a84aa8985b3d29e1eee7e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733861773247 2024-12-10T20:16:16,038 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 576bed617d7e4ea2a6dd78803e867b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733861773670 2024-12-10T20:16:16,038 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea792f3a3eab4371aac1f96f08bea4d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733861774375 2024-12-10T20:16:16,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742139_1315 (size=12301) 2024-12-10T20:16:16,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/74ff9ad9b1da4b7794617dc27746033b 2024-12-10T20:16:16,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861836056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861836061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,075 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:16,076 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/cc751339f7084be8816df15047dc481f is 50, key is test_row_0/C:col10/1733861774731/Put/seqid=0 2024-12-10T20:16:16,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ccc8d43dc1c8430f982da574fd3e7dbf is 50, key is test_row_0/B:col10/1733861774806/Put/seqid=0 2024-12-10T20:16:16,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742140_1316 (size=13119) 2024-12-10T20:16:16,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742141_1317 (size=12301) 2024-12-10T20:16:16,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ccc8d43dc1c8430f982da574fd3e7dbf 2024-12-10T20:16:16,166 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/cc751339f7084be8816df15047dc481f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/cc751339f7084be8816df15047dc481f 2024-12-10T20:16:16,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c2c269efc4a74fa68836346fdd47d9b7 is 50, key is test_row_0/C:col10/1733861774806/Put/seqid=0 2024-12-10T20:16:16,180 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into cc751339f7084be8816df15047dc481f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:16,180 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:16,180 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861775943; duration=0sec 2024-12-10T20:16:16,180 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:16,180 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861836169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861836188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742142_1318 (size=12301) 2024-12-10T20:16:16,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861836387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:16,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:16,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861836408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,424 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/2258e28340dd445eb74b78b93d5bd480 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/2258e28340dd445eb74b78b93d5bd480 2024-12-10T20:16:16,443 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 2258e28340dd445eb74b78b93d5bd480(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:16,443 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:16,443 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861775943; duration=0sec 2024-12-10T20:16:16,443 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:16,443 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:16,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c2c269efc4a74fa68836346fdd47d9b7 2024-12-10T20:16:16,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/74ff9ad9b1da4b7794617dc27746033b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/74ff9ad9b1da4b7794617dc27746033b 2024-12-10T20:16:16,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/74ff9ad9b1da4b7794617dc27746033b, entries=150, sequenceid=371, filesize=12.0 K 2024-12-10T20:16:16,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/ccc8d43dc1c8430f982da574fd3e7dbf as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ccc8d43dc1c8430f982da574fd3e7dbf 2024-12-10T20:16:16,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ccc8d43dc1c8430f982da574fd3e7dbf, entries=150, sequenceid=371, filesize=12.0 K 2024-12-10T20:16:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/c2c269efc4a74fa68836346fdd47d9b7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c269efc4a74fa68836346fdd47d9b7 2024-12-10T20:16:16,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c269efc4a74fa68836346fdd47d9b7, entries=150, sequenceid=371, filesize=12.0 K 2024-12-10T20:16:16,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d574f3055e0a5aa98dce8c43f5c73a8d in 675ms, sequenceid=371, compaction requested=false 2024-12-10T20:16:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:16,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:16:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:16,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/35bb4c27e5904ea8a08a3b3c88a6144d is 50, key is test_row_0/A:col10/1733861776050/Put/seqid=0 2024-12-10T20:16:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742143_1319 (size=12301) 2024-12-10T20:16:16,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861836759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861836764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861836877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861836877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:16,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:16,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:17,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:17,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861837087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861837087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/35bb4c27e5904ea8a08a3b3c88a6144d 2024-12-10T20:16:17,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:17,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:17,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/6ef08dbff5c44d2395d362387da215f4 is 50, key is test_row_0/B:col10/1733861776050/Put/seqid=0 2024-12-10T20:16:17,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742144_1320 (size=12301) 2024-12-10T20:16:17,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/6ef08dbff5c44d2395d362387da215f4 2024-12-10T20:16:17,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e384ef94c17e472aa8390d7e06dcabb8 is 50, key is test_row_0/C:col10/1733861776050/Put/seqid=0 2024-12-10T20:16:17,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742145_1321 (size=12301) 2024-12-10T20:16:17,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e384ef94c17e472aa8390d7e06dcabb8 2024-12-10T20:16:17,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/35bb4c27e5904ea8a08a3b3c88a6144d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/35bb4c27e5904ea8a08a3b3c88a6144d 2024-12-10T20:16:17,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/35bb4c27e5904ea8a08a3b3c88a6144d, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T20:16:17,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/6ef08dbff5c44d2395d362387da215f4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6ef08dbff5c44d2395d362387da215f4 2024-12-10T20:16:17,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6ef08dbff5c44d2395d362387da215f4, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T20:16:17,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e384ef94c17e472aa8390d7e06dcabb8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e384ef94c17e472aa8390d7e06dcabb8 2024-12-10T20:16:17,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e384ef94c17e472aa8390d7e06dcabb8, entries=150, sequenceid=391, filesize=12.0 K 2024-12-10T20:16:17,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d574f3055e0a5aa98dce8c43f5c73a8d in 636ms, sequenceid=391, compaction requested=true 2024-12-10T20:16:17,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:17,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:17,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:17,350 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:17,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:17,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:17,350 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:17,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:17,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:17,357 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:17,357 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:17,357 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:17,357 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:17,357 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,357 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/269d374a2ca345b386ea248be3aba5b9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/74ff9ad9b1da4b7794617dc27746033b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/35bb4c27e5904ea8a08a3b3c88a6144d] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.8 K 2024-12-10T20:16:17,357 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,357 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/2258e28340dd445eb74b78b93d5bd480, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ccc8d43dc1c8430f982da574fd3e7dbf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6ef08dbff5c44d2395d362387da215f4] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.8 K 2024-12-10T20:16:17,358 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 269d374a2ca345b386ea248be3aba5b9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733861774375 2024-12-10T20:16:17,358 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 2258e28340dd445eb74b78b93d5bd480, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733861774375 2024-12-10T20:16:17,359 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ff9ad9b1da4b7794617dc27746033b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733861774806 2024-12-10T20:16:17,359 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc8d43dc1c8430f982da574fd3e7dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733861774806 2024-12-10T20:16:17,359 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef08dbff5c44d2395d362387da215f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733861776050 2024-12-10T20:16:17,359 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35bb4c27e5904ea8a08a3b3c88a6144d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733861776050 2024-12-10T20:16:17,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T20:16:17,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,382 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:16:17,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:17,383 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:17,383 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/698e590fe9624917ba473e1d45dfc07f is 50, key is test_row_0/B:col10/1733861776050/Put/seqid=0 2024-12-10T20:16:17,394 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:17,395 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/394fb1b75ed04c4eaf8aba2e409708f2 is 50, key is test_row_0/A:col10/1733861776050/Put/seqid=0 2024-12-10T20:16:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:17,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:17,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3f0f070f94c14602a9d7afd41f34a336 is 50, key is test_row_0/A:col10/1733861776759/Put/seqid=0 2024-12-10T20:16:17,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742146_1322 (size=13221) 2024-12-10T20:16:17,431 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/698e590fe9624917ba473e1d45dfc07f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/698e590fe9624917ba473e1d45dfc07f 2024-12-10T20:16:17,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742147_1323 (size=13221) 2024-12-10T20:16:17,447 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 698e590fe9624917ba473e1d45dfc07f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:17,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:17,447 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861777350; duration=0sec 2024-12-10T20:16:17,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:17,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:17,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:17,455 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:17,455 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:17,455 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,455 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/cc751339f7084be8816df15047dc481f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c269efc4a74fa68836346fdd47d9b7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e384ef94c17e472aa8390d7e06dcabb8] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.8 K 2024-12-10T20:16:17,456 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting cc751339f7084be8816df15047dc481f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733861774375 2024-12-10T20:16:17,456 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c269efc4a74fa68836346fdd47d9b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733861774806 2024-12-10T20:16:17,457 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e384ef94c17e472aa8390d7e06dcabb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733861776050 2024-12-10T20:16:17,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742148_1324 (size=12301) 2024-12-10T20:16:17,458 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/394fb1b75ed04c4eaf8aba2e409708f2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/394fb1b75ed04c4eaf8aba2e409708f2 2024-12-10T20:16:17,458 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3f0f070f94c14602a9d7afd41f34a336 2024-12-10T20:16:17,470 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 394fb1b75ed04c4eaf8aba2e409708f2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:17,470 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:17,471 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861777350; duration=0sec 2024-12-10T20:16:17,471 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:17,471 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/7c783680760b4986b4b4989d0b22af61 is 50, key is test_row_0/B:col10/1733861776759/Put/seqid=0 2024-12-10T20:16:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861837476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861837476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742149_1325 (size=12301) 2024-12-10T20:16:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,491 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/7c783680760b4986b4b4989d0b22af61 2024-12-10T20:16:17,498 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:17,498 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6e89dc1137fb4118b3ddfd3dd3b5ea41 is 50, key is test_row_0/C:col10/1733861776050/Put/seqid=0 2024-12-10T20:16:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742150_1326 (size=13221) 2024-12-10T20:16:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e64d72ed71144a969128ad8951898cc5 is 50, key is test_row_0/C:col10/1733861776759/Put/seqid=0 2024-12-10T20:16:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742151_1327 (size=12301) 2024-12-10T20:16:17,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861837601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861837604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:16:17,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861837818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861837818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:17,941 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6e89dc1137fb4118b3ddfd3dd3b5ea41 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6e89dc1137fb4118b3ddfd3dd3b5ea41 2024-12-10T20:16:17,962 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e64d72ed71144a969128ad8951898cc5 2024-12-10T20:16:17,963 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 6e89dc1137fb4118b3ddfd3dd3b5ea41(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:17,963 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:17,963 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861777350; duration=0sec 2024-12-10T20:16:17,963 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:17,963 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:17,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3f0f070f94c14602a9d7afd41f34a336 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3f0f070f94c14602a9d7afd41f34a336 2024-12-10T20:16:17,981 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3f0f070f94c14602a9d7afd41f34a336, entries=150, sequenceid=406, filesize=12.0 K 2024-12-10T20:16:17,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/7c783680760b4986b4b4989d0b22af61 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7c783680760b4986b4b4989d0b22af61 2024-12-10T20:16:17,987 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7c783680760b4986b4b4989d0b22af61, entries=150, sequenceid=406, filesize=12.0 K 2024-12-10T20:16:17,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/e64d72ed71144a969128ad8951898cc5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e64d72ed71144a969128ad8951898cc5 2024-12-10T20:16:17,992 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e64d72ed71144a969128ad8951898cc5, entries=150, sequenceid=406, filesize=12.0 K 2024-12-10T20:16:17,994 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for d574f3055e0a5aa98dce8c43f5c73a8d in 612ms, sequenceid=406, compaction requested=false 2024-12-10T20:16:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-10T20:16:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-10T20:16:17,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-10T20:16:17,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3350 sec 2024-12-10T20:16:18,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 4.3420 sec 2024-12-10T20:16:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:18,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T20:16:18,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861777466, max timestamp=1733861778143Num uniques 859; 2024-12-10T20:16:18,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861777466, max timestamp=1733861778143Num uniques 859; 2024-12-10T20:16:18,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:18,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:18,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/9a0fde8275ab4d5ca865e867b84a8eb3 is 50, key is test_row_0/A:col10/1733861778143/Put/seqid=0 2024-12-10T20:16:18,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742152_1328 (size=12301) 2024-12-10T20:16:18,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861778189, max timestamp=1733861778201Num uniques 859; 2024-12-10T20:16:18,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861778189, max timestamp=1733861778201Num uniques 859; 2024-12-10T20:16:18,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:18,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:18,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861778189, max timestamp=1733861778201Num uniques 859; 2024-12-10T20:16:18,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861838252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861838253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861838367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861838367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861838577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861838577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/9a0fde8275ab4d5ca865e867b84a8eb3 2024-12-10T20:16:18,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c5fccdf950bb4c6798409586c31fee49 is 50, key is test_row_0/B:col10/1733861778143/Put/seqid=0 2024-12-10T20:16:18,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742153_1329 (size=12301) 2024-12-10T20:16:18,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861838887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:18,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861838888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:19,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c5fccdf950bb4c6798409586c31fee49 2024-12-10T20:16:19,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/9083f10774394682b52d2347b636d63c is 50, key is test_row_0/C:col10/1733861778143/Put/seqid=0 2024-12-10T20:16:19,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742154_1330 (size=12301) 2024-12-10T20:16:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861839408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861839408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:19,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/9083f10774394682b52d2347b636d63c 2024-12-10T20:16:19,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/9a0fde8275ab4d5ca865e867b84a8eb3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9a0fde8275ab4d5ca865e867b84a8eb3 2024-12-10T20:16:19,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9a0fde8275ab4d5ca865e867b84a8eb3, entries=150, sequenceid=430, filesize=12.0 K 2024-12-10T20:16:19,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/c5fccdf950bb4c6798409586c31fee49 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c5fccdf950bb4c6798409586c31fee49 2024-12-10T20:16:19,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c5fccdf950bb4c6798409586c31fee49, entries=150, sequenceid=430, filesize=12.0 K 2024-12-10T20:16:19,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/9083f10774394682b52d2347b636d63c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9083f10774394682b52d2347b636d63c 2024-12-10T20:16:19,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9083f10774394682b52d2347b636d63c, entries=150, sequenceid=430, filesize=12.0 K 2024-12-10T20:16:19,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for d574f3055e0a5aa98dce8c43f5c73a8d in 1471ms, sequenceid=430, compaction requested=true 2024-12-10T20:16:19,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:19,616 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:16:19,616 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:19,617 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:19,617 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:19,617 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/394fb1b75ed04c4eaf8aba2e409708f2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3f0f070f94c14602a9d7afd41f34a336, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9a0fde8275ab4d5ca865e867b84a8eb3] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.9 K 2024-12-10T20:16:19,617 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:19,617 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 394fb1b75ed04c4eaf8aba2e409708f2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733861776050 2024-12-10T20:16:19,618 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f0f070f94c14602a9d7afd41f34a336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733861776750 2024-12-10T20:16:19,618 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:19,618 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:19,619 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:19,619 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/698e590fe9624917ba473e1d45dfc07f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7c783680760b4986b4b4989d0b22af61, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c5fccdf950bb4c6798409586c31fee49] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.9 K 2024-12-10T20:16:19,619 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0fde8275ab4d5ca865e867b84a8eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861777466 2024-12-10T20:16:19,620 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 698e590fe9624917ba473e1d45dfc07f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733861776050 2024-12-10T20:16:19,622 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c783680760b4986b4b4989d0b22af61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733861776750 2024-12-10T20:16:19,623 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c5fccdf950bb4c6798409586c31fee49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861777466 2024-12-10T20:16:19,645 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:19,645 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/ba1c3b4c2f2546f49e9d5e609acea9d6 is 50, key is test_row_0/A:col10/1733861778143/Put/seqid=0 2024-12-10T20:16:19,654 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:19,655 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/0ae2b66604aa455fb3a4cfee5fb582ea is 50, key is test_row_0/B:col10/1733861778143/Put/seqid=0 2024-12-10T20:16:19,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742155_1331 (size=13323) 2024-12-10T20:16:19,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742156_1332 (size=13323) 2024-12-10T20:16:19,680 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/0ae2b66604aa455fb3a4cfee5fb582ea as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0ae2b66604aa455fb3a4cfee5fb582ea 2024-12-10T20:16:19,680 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/ba1c3b4c2f2546f49e9d5e609acea9d6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ba1c3b4c2f2546f49e9d5e609acea9d6 2024-12-10T20:16:19,685 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into ba1c3b4c2f2546f49e9d5e609acea9d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:19,685 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:19,685 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861779615; duration=0sec 2024-12-10T20:16:19,685 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:19,685 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:19,685 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:19,687 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:19,687 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:19,687 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:19,688 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6e89dc1137fb4118b3ddfd3dd3b5ea41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e64d72ed71144a969128ad8951898cc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9083f10774394682b52d2347b636d63c] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=36.9 K 2024-12-10T20:16:19,688 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e89dc1137fb4118b3ddfd3dd3b5ea41, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733861776050 2024-12-10T20:16:19,689 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e64d72ed71144a969128ad8951898cc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733861776750 2024-12-10T20:16:19,691 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9083f10774394682b52d2347b636d63c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861777466 2024-12-10T20:16:19,691 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 0ae2b66604aa455fb3a4cfee5fb582ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:19,691 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:19,691 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861779616; duration=0sec 2024-12-10T20:16:19,692 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:19,692 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:19,706 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:19,707 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/9aacc4c51d7c4593a1ff2a87796f31fb is 50, key is test_row_0/C:col10/1733861778143/Put/seqid=0 2024-12-10T20:16:19,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742157_1333 (size=13323) 2024-12-10T20:16:19,744 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/9aacc4c51d7c4593a1ff2a87796f31fb as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9aacc4c51d7c4593a1ff2a87796f31fb 2024-12-10T20:16:19,751 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 9aacc4c51d7c4593a1ff2a87796f31fb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:19,751 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:19,751 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861779616; duration=0sec 2024-12-10T20:16:19,751 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:19,752 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:20,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T20:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:20,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:20,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/8e5c976475384946ad681314b0a6fc6e is 50, key is test_row_0/A:col10/1733861778242/Put/seqid=0 2024-12-10T20:16:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861840460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861840461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742158_1334 (size=14741) 2024-12-10T20:16:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861840577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:20,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861840577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:20,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861840787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:20,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861840787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:20,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:20,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/8e5c976475384946ad681314b0a6fc6e 2024-12-10T20:16:20,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/392a6bfd27bc4a2ea139d04de6db5c1b is 50, key is test_row_0/B:col10/1733861778242/Put/seqid=0 2024-12-10T20:16:20,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742159_1335 (size=12301) 2024-12-10T20:16:20,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/392a6bfd27bc4a2ea139d04de6db5c1b 2024-12-10T20:16:21,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/26a88e6ef2c942d08e4518e66d7293ec is 50, key is test_row_0/C:col10/1733861778242/Put/seqid=0 2024-12-10T20:16:21,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742160_1336 (size=12301) 2024-12-10T20:16:21,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/26a88e6ef2c942d08e4518e66d7293ec 2024-12-10T20:16:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/8e5c976475384946ad681314b0a6fc6e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8e5c976475384946ad681314b0a6fc6e 2024-12-10T20:16:21,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8e5c976475384946ad681314b0a6fc6e, entries=200, sequenceid=470, filesize=14.4 K 2024-12-10T20:16:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861841101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861841102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:21,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/392a6bfd27bc4a2ea139d04de6db5c1b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/392a6bfd27bc4a2ea139d04de6db5c1b 2024-12-10T20:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:21,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/392a6bfd27bc4a2ea139d04de6db5c1b, entries=150, sequenceid=470, filesize=12.0 K 2024-12-10T20:16:21,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/26a88e6ef2c942d08e4518e66d7293ec as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/26a88e6ef2c942d08e4518e66d7293ec 2024-12-10T20:16:21,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/26a88e6ef2c942d08e4518e66d7293ec, entries=150, sequenceid=470, filesize=12.0 K 2024-12-10T20:16:21,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for d574f3055e0a5aa98dce8c43f5c73a8d in 719ms, sequenceid=470, compaction requested=false 2024-12-10T20:16:21,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:21,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:16:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:21,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:21,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3dad1f1d711b4450b1bf79ceed51bdbf is 50, key is test_row_0/A:col10/1733861781619/Put/seqid=0 2024-12-10T20:16:21,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742161_1337 (size=14741) 2024-12-10T20:16:21,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861781638, max timestamp=1733861781736Num uniques 859; 2024-12-10T20:16:21,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861781638, max timestamp=1733861781736Num uniques 859; 2024-12-10T20:16:21,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:21,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:21,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861781638, max timestamp=1733861781736Num uniques 859; 2024-12-10T20:16:21,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T20:16:21,845 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:21,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:21,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861841835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-10T20:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T20:16:21,848 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:21,849 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:21,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:21,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861841846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T20:16:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861841957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861841972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:22,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T20:16:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3dad1f1d711b4450b1bf79ceed51bdbf 2024-12-10T20:16:22,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/229e96038bd04cc7bdc5830c4088faba is 50, key is test_row_0/B:col10/1733861781619/Put/seqid=0 2024-12-10T20:16:22,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742162_1338 (size=12301) 2024-12-10T20:16:22,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T20:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/229e96038bd04cc7bdc5830c4088faba 2024-12-10T20:16:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T20:16:22,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861842167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:22,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/2b8e555af50c466f85ef2180bf431309 is 50, key is test_row_0/C:col10/1733861781619/Put/seqid=0 2024-12-10T20:16:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861842187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:22,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742163_1339 (size=12301) 2024-12-10T20:16:22,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T20:16:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T20:16:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T20:16:22,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861842496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861842496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:22,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T20:16:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:22,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/2b8e555af50c466f85ef2180bf431309 2024-12-10T20:16:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/3dad1f1d711b4450b1bf79ceed51bdbf as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3dad1f1d711b4450b1bf79ceed51bdbf 2024-12-10T20:16:22,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3dad1f1d711b4450b1bf79ceed51bdbf, entries=200, sequenceid=481, filesize=14.4 K 2024-12-10T20:16:22,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/229e96038bd04cc7bdc5830c4088faba as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/229e96038bd04cc7bdc5830c4088faba 2024-12-10T20:16:22,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/229e96038bd04cc7bdc5830c4088faba, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T20:16:22,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/2b8e555af50c466f85ef2180bf431309 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/2b8e555af50c466f85ef2180bf431309 2024-12-10T20:16:22,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/2b8e555af50c466f85ef2180bf431309, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T20:16:22,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d574f3055e0a5aa98dce8c43f5c73a8d in 1067ms, sequenceid=481, compaction requested=true 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:22,688 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:22,688 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:22,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:22,690 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:22,690 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:22,690 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,690 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ba1c3b4c2f2546f49e9d5e609acea9d6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8e5c976475384946ad681314b0a6fc6e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3dad1f1d711b4450b1bf79ceed51bdbf] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=41.8 K 2024-12-10T20:16:22,691 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:22,691 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1c3b4c2f2546f49e9d5e609acea9d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861777466 2024-12-10T20:16:22,691 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:22,692 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,692 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0ae2b66604aa455fb3a4cfee5fb582ea, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/392a6bfd27bc4a2ea139d04de6db5c1b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/229e96038bd04cc7bdc5830c4088faba] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.0 K 2024-12-10T20:16:22,692 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5c976475384946ad681314b0a6fc6e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733861778242 2024-12-10T20:16:22,692 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae2b66604aa455fb3a4cfee5fb582ea, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861777466 2024-12-10T20:16:22,692 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dad1f1d711b4450b1bf79ceed51bdbf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733861780451 2024-12-10T20:16:22,693 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 392a6bfd27bc4a2ea139d04de6db5c1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733861778242 2024-12-10T20:16:22,694 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 229e96038bd04cc7bdc5830c4088faba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733861780451 2024-12-10T20:16:22,706 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:22,706 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/51db4dcaf8b340458f37ad0d1fb9efb5 is 50, key is test_row_0/B:col10/1733861781619/Put/seqid=0 2024-12-10T20:16:22,721 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:22,721 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/b61c735df6504099b3511e40bf05cde2 is 50, key is test_row_0/A:col10/1733861781619/Put/seqid=0 2024-12-10T20:16:22,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742164_1340 (size=13425) 2024-12-10T20:16:22,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742165_1341 (size=13425) 2024-12-10T20:16:22,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T20:16:22,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,786 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-10T20:16:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:22,797 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/b61c735df6504099b3511e40bf05cde2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b61c735df6504099b3511e40bf05cde2 2024-12-10T20:16:22,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/8900411d22ca44f6aa53900ea4fa85dd is 50, key is test_row_0/A:col10/1733861781831/Put/seqid=0 2024-12-10T20:16:22,813 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into b61c735df6504099b3511e40bf05cde2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:22,813 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:22,813 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861782688; duration=0sec 2024-12-10T20:16:22,813 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:22,813 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:22,813 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:22,815 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:22,815 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:22,816 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:22,816 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9aacc4c51d7c4593a1ff2a87796f31fb, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/26a88e6ef2c942d08e4518e66d7293ec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/2b8e555af50c466f85ef2180bf431309] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.0 K 2024-12-10T20:16:22,816 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aacc4c51d7c4593a1ff2a87796f31fb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733861777466 2024-12-10T20:16:22,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742166_1342 (size=12301) 2024-12-10T20:16:22,818 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/8900411d22ca44f6aa53900ea4fa85dd 2024-12-10T20:16:22,824 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a88e6ef2c942d08e4518e66d7293ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733861778242 2024-12-10T20:16:22,825 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8e555af50c466f85ef2180bf431309, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733861780451 2024-12-10T20:16:22,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/7937e158c28b4a7a8387f726b392dae8 is 50, key is test_row_0/B:col10/1733861781831/Put/seqid=0 2024-12-10T20:16:22,863 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#298 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:22,865 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/1f804e7260524be5bf7c955d5df09dbd is 50, key is test_row_0/C:col10/1733861781619/Put/seqid=0 2024-12-10T20:16:22,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742167_1343 (size=12301) 2024-12-10T20:16:22,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742168_1344 (size=13425) 2024-12-10T20:16:22,937 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/1f804e7260524be5bf7c955d5df09dbd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/1f804e7260524be5bf7c955d5df09dbd 2024-12-10T20:16:22,956 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 1f804e7260524be5bf7c955d5df09dbd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:22,956 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:22,956 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861782688; duration=0sec 2024-12-10T20:16:22,956 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:22,956 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:22,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T20:16:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861843020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861843021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861843024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60586 deadline: 1733861843048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,052 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18325 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60584 deadline: 1733861843087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,091 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in d574f3055e0a5aa98dce8c43f5c73a8d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18363 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861843137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861843137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861843138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,168 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/51db4dcaf8b340458f37ad0d1fb9efb5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/51db4dcaf8b340458f37ad0d1fb9efb5 2024-12-10T20:16:23,176 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 51db4dcaf8b340458f37ad0d1fb9efb5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:23,176 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:23,176 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861782688; duration=0sec 2024-12-10T20:16:23,176 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:23,176 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:23,294 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/7937e158c28b4a7a8387f726b392dae8 2024-12-10T20:16:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/65eea3238ca64bfd96a8ca1fe6d2c4ad is 50, key is test_row_0/C:col10/1733861781831/Put/seqid=0 2024-12-10T20:16:23,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742169_1345 (size=12301) 2024-12-10T20:16:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861843346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861843347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861843347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60656 deadline: 1733861843657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60630 deadline: 1733861843657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60606 deadline: 1733861843657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:16:23,727 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/65eea3238ca64bfd96a8ca1fe6d2c4ad 2024-12-10T20:16:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/8900411d22ca44f6aa53900ea4fa85dd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8900411d22ca44f6aa53900ea4fa85dd 2024-12-10T20:16:23,736 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8900411d22ca44f6aa53900ea4fa85dd, entries=150, sequenceid=516, filesize=12.0 K 2024-12-10T20:16:23,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/7937e158c28b4a7a8387f726b392dae8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7937e158c28b4a7a8387f726b392dae8 2024-12-10T20:16:23,740 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7937e158c28b4a7a8387f726b392dae8, entries=150, sequenceid=516, filesize=12.0 K 2024-12-10T20:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/65eea3238ca64bfd96a8ca1fe6d2c4ad as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/65eea3238ca64bfd96a8ca1fe6d2c4ad 2024-12-10T20:16:23,744 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/65eea3238ca64bfd96a8ca1fe6d2c4ad, entries=150, sequenceid=516, filesize=12.0 K 2024-12-10T20:16:23,745 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d574f3055e0a5aa98dce8c43f5c73a8d in 959ms, sequenceid=516, compaction requested=false 2024-12-10T20:16:23,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:23,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:23,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-10T20:16:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-10T20:16:23,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-10T20:16:23,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-12-10T20:16:23,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9020 sec 2024-12-10T20:16:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T20:16:24,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:24,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-10T20:16:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T20:16:24,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:24,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:24,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:24,020 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-12-10T20:16:24,020 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-12-10T20:16:24,020 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,020 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,020 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,020 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,020 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,020 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,021 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-12-10T20:16:24,021 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,021 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,021 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,022 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-12-10T20:16:24,022 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,022 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,022 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,023 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-12-10T20:16:24,023 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,023 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,023 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T20:16:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:24,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T20:16:24,170 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-12-10T20:16:24,170 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-12-10T20:16:24,170 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-12-10T20:16:24,170 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,170 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,170 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:24,170 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,170 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:24,170 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:24,170 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,170 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,170 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:24,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:24,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:24,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:24,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:24,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:24,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/7c6ee5b1fe174670b49a576000fe0394 is 50, key is test_row_0/A:col10/1733861784168/Put/seqid=0 2024-12-10T20:16:24,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742170_1346 (size=12301) 2024-12-10T20:16:24,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T20:16:24,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:24,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/7c6ee5b1fe174670b49a576000fe0394 2024-12-10T20:16:24,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/35f0af8044a141dd872c31404a9aac76 is 50, key is test_row_0/B:col10/1733861784168/Put/seqid=0 2024-12-10T20:16:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742171_1347 (size=12301) 2024-12-10T20:16:24,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:24,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:24,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T20:16:24,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:24,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:24,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/35f0af8044a141dd872c31404a9aac76 2024-12-10T20:16:24,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6b1ac66474ed4b938ffaa30aa8fb51e7 is 50, key is test_row_0/C:col10/1733861784168/Put/seqid=0 2024-12-10T20:16:25,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742172_1348 (size=12301) 2024-12-10T20:16:25,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T20:16:25,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6b1ac66474ed4b938ffaa30aa8fb51e7 2024-12-10T20:16:25,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. as already flushing 2024-12-10T20:16:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/7c6ee5b1fe174670b49a576000fe0394 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7c6ee5b1fe174670b49a576000fe0394 2024-12-10T20:16:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:25,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7c6ee5b1fe174670b49a576000fe0394, entries=150, sequenceid=533, filesize=12.0 K 2024-12-10T20:16:25,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/35f0af8044a141dd872c31404a9aac76 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/35f0af8044a141dd872c31404a9aac76 2024-12-10T20:16:25,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/35f0af8044a141dd872c31404a9aac76, entries=150, sequenceid=533, filesize=12.0 K 2024-12-10T20:16:25,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/6b1ac66474ed4b938ffaa30aa8fb51e7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6b1ac66474ed4b938ffaa30aa8fb51e7 2024-12-10T20:16:25,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6b1ac66474ed4b938ffaa30aa8fb51e7, entries=150, sequenceid=533, filesize=12.0 K 2024-12-10T20:16:25,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for d574f3055e0a5aa98dce8c43f5c73a8d in 1249ms, sequenceid=533, compaction requested=true 2024-12-10T20:16:25,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:25,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:25,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:25,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:25,420 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:25,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:25,420 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:25,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d574f3055e0a5aa98dce8c43f5c73a8d:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:25,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:25,420 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:25,420 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/B is initiating minor compaction (all files) 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/A is initiating minor compaction (all files) 2024-12-10T20:16:25,421 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/B in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,421 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/A in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,421 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/51db4dcaf8b340458f37ad0d1fb9efb5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7937e158c28b4a7a8387f726b392dae8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/35f0af8044a141dd872c31404a9aac76] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.1 K 2024-12-10T20:16:25,421 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b61c735df6504099b3511e40bf05cde2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8900411d22ca44f6aa53900ea4fa85dd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7c6ee5b1fe174670b49a576000fe0394] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.1 K 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 51db4dcaf8b340458f37ad0d1fb9efb5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733861780451 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61c735df6504099b3511e40bf05cde2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733861780451 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8900411d22ca44f6aa53900ea4fa85dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733861781821 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7937e158c28b4a7a8387f726b392dae8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733861781821 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6ee5b1fe174670b49a576000fe0394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733861782987 2024-12-10T20:16:25,421 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f0af8044a141dd872c31404a9aac76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733861782987 2024-12-10T20:16:25,427 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:25,427 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:25,428 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/708ba96d53ea48d69d069e2e7bdb6485 is 50, key is test_row_0/A:col10/1733861784168/Put/seqid=0 2024-12-10T20:16:25,428 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/1e795929a0304b658e4deaa6cd8d10ad is 50, key is test_row_0/B:col10/1733861784168/Put/seqid=0 2024-12-10T20:16:25,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742174_1350 (size=13527) 2024-12-10T20:16:25,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742173_1349 (size=13527) 2024-12-10T20:16:25,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T20:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-10T20:16:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-10T20:16:25,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-10T20:16:25,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-12-10T20:16:25,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5440 sec 2024-12-10T20:16:25,841 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/1e795929a0304b658e4deaa6cd8d10ad as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1e795929a0304b658e4deaa6cd8d10ad 2024-12-10T20:16:25,841 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/708ba96d53ea48d69d069e2e7bdb6485 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/708ba96d53ea48d69d069e2e7bdb6485 2024-12-10T20:16:25,846 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/A of d574f3055e0a5aa98dce8c43f5c73a8d into 708ba96d53ea48d69d069e2e7bdb6485(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:25,846 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/B of d574f3055e0a5aa98dce8c43f5c73a8d into 1e795929a0304b658e4deaa6cd8d10ad(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:25,847 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/B, priority=13, startTime=1733861785420; duration=0sec 2024-12-10T20:16:25,847 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/A, priority=13, startTime=1733861785419; duration=0sec 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:A 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:B 2024-12-10T20:16:25,847 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:25,848 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:25,848 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): d574f3055e0a5aa98dce8c43f5c73a8d/C is initiating minor compaction (all files) 2024-12-10T20:16:25,848 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d574f3055e0a5aa98dce8c43f5c73a8d/C in TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:25,848 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/1f804e7260524be5bf7c955d5df09dbd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/65eea3238ca64bfd96a8ca1fe6d2c4ad, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6b1ac66474ed4b938ffaa30aa8fb51e7] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp, totalSize=37.1 K 2024-12-10T20:16:25,848 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f804e7260524be5bf7c955d5df09dbd, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733861780451 2024-12-10T20:16:25,849 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65eea3238ca64bfd96a8ca1fe6d2c4ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1733861781821 2024-12-10T20:16:25,849 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1ac66474ed4b938ffaa30aa8fb51e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733861782987 2024-12-10T20:16:25,856 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d574f3055e0a5aa98dce8c43f5c73a8d#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:25,856 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/83e5ac0dec364442a5327d8f934d96dd is 50, key is test_row_0/C:col10/1733861784168/Put/seqid=0 2024-12-10T20:16:25,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742175_1351 (size=13527) 2024-12-10T20:16:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T20:16:26,178 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:26,267 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/83e5ac0dec364442a5327d8f934d96dd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/83e5ac0dec364442a5327d8f934d96dd 2024-12-10T20:16:26,274 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d574f3055e0a5aa98dce8c43f5c73a8d/C of d574f3055e0a5aa98dce8c43f5c73a8d into 83e5ac0dec364442a5327d8f934d96dd(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:26,274 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d574f3055e0a5aa98dce8c43f5c73a8d: 2024-12-10T20:16:26,274 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., storeName=d574f3055e0a5aa98dce8c43f5c73a8d/C, priority=13, startTime=1733861785420; duration=0sec 2024-12-10T20:16:26,274 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:26,275 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d574f3055e0a5aa98dce8c43f5c73a8d:C 2024-12-10T20:16:33,069 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-12-10T20:16:33,069 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:33,069 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:33,069 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:33,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:33,159 INFO [Thread-1127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1127. 2024-12-10T20:16:33,159 DEBUG [Thread-1127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:33,159 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:33,159 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:33,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 209 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2191 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2257 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2212 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2203 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2249 2024-12-10T20:16:33,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T20:16:33,159 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:16:33,160 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:16:33,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:33,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:33,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:33,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T20:16:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T20:16:33,165 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861793165"}]},"ts":"1733861793165"} 2024-12-10T20:16:33,167 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T20:16:33,167 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T20:16:33,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:16:33,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, UNASSIGN}] 2024-12-10T20:16:33,170 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, UNASSIGN 2024-12-10T20:16:33,171 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=d574f3055e0a5aa98dce8c43f5c73a8d, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:33,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, UNASSIGN because future has completed 2024-12-10T20:16:33,174 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:16:33,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:16:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T20:16:33,327 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing d574f3055e0a5aa98dce8c43f5c73a8d, disabling compactions & flushes 2024-12-10T20:16:33,327 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. after waiting 0 ms 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:33,327 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing d574f3055e0a5aa98dce8c43f5c73a8d 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=A 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:33,327 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=B 2024-12-10T20:16:33,328 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:33,328 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d574f3055e0a5aa98dce8c43f5c73a8d, store=C 2024-12-10T20:16:33,328 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:33,331 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/cc0c8dcdcfd14adfa0131d9940396731 is 50, key is test_row_0/A:col10/1733861793067/Put/seqid=0 2024-12-10T20:16:33,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742176_1352 (size=9857) 2024-12-10T20:16:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T20:16:33,734 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/cc0c8dcdcfd14adfa0131d9940396731 2024-12-10T20:16:33,740 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d1d7263f4be14120910ccc5b3327e3a1 is 50, key is test_row_0/B:col10/1733861793067/Put/seqid=0 2024-12-10T20:16:33,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742177_1353 (size=9857) 2024-12-10T20:16:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T20:16:34,145 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d1d7263f4be14120910ccc5b3327e3a1 2024-12-10T20:16:34,154 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/3eea98e654a049bb9efb1f3739b1075f is 50, key is test_row_0/C:col10/1733861793067/Put/seqid=0 2024-12-10T20:16:34,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742178_1354 (size=9857) 2024-12-10T20:16:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T20:16:34,559 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/3eea98e654a049bb9efb1f3739b1075f 2024-12-10T20:16:34,564 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/A/cc0c8dcdcfd14adfa0131d9940396731 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cc0c8dcdcfd14adfa0131d9940396731 2024-12-10T20:16:34,568 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cc0c8dcdcfd14adfa0131d9940396731, entries=100, sequenceid=542, filesize=9.6 K 2024-12-10T20:16:34,570 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/B/d1d7263f4be14120910ccc5b3327e3a1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1d7263f4be14120910ccc5b3327e3a1 2024-12-10T20:16:34,575 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1d7263f4be14120910ccc5b3327e3a1, entries=100, sequenceid=542, filesize=9.6 K 2024-12-10T20:16:34,576 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/.tmp/C/3eea98e654a049bb9efb1f3739b1075f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/3eea98e654a049bb9efb1f3739b1075f 2024-12-10T20:16:34,580 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/3eea98e654a049bb9efb1f3739b1075f, entries=100, sequenceid=542, filesize=9.6 K 2024-12-10T20:16:34,581 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for d574f3055e0a5aa98dce8c43f5c73a8d in 1254ms, sequenceid=542, compaction requested=false 2024-12-10T20:16:34,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f363aee6d03841be985d37d364eb53f7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/c730f0d968f749478c7e515ee6daba09, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cb6d8a0fe2734826a321e95ac1193788, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/dc653801d6bc4322b051ede096c0d62f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/73fe05a11c124c95a493448b04b4ff72, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a75f65a66d346b8a31e162c863023f6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7ae3d44ddd0a4c4b94a5414be0aade9a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f392f6cad38a4faa9e0c9fb1d5180867, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b623132421a0429fa5f029421b83caf9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f1938b81c6ab47e8b34a82e3c8d4aa5f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3ee3dc5de0a041cfb52b6f033353ce9b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4f812cdeae0a400c9e877c05a64d71e1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/57e7164e08394b9296689afcaf74ff64, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9d45bc5b3b05494ba371580ab72c1c68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/205c5bcb4f1e4eb590b1f2f4871a847a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/93e59093f42d4c67b3b09e9672a6ee36, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/29723507b50d4f45bc37df60c5f14b7e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/23c0b6ed7cfa4ec88acb8c243d82ffd4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b379e486d03413db1424490cdcfeb72, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ea64870ca2644a1393855d9d8edadf67, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b7badbebcfa439db6aea8d46f4eba56, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4b4569c458fa4a9b9dd594a24846dffc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a19bafd5d504cd39243cc1d943c351c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/d22ceefab8114bfdbecbcddaf3ae3ff2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/269d374a2ca345b386ea248be3aba5b9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/74ff9ad9b1da4b7794617dc27746033b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/394fb1b75ed04c4eaf8aba2e409708f2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/35bb4c27e5904ea8a08a3b3c88a6144d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3f0f070f94c14602a9d7afd41f34a336, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ba1c3b4c2f2546f49e9d5e609acea9d6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9a0fde8275ab4d5ca865e867b84a8eb3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8e5c976475384946ad681314b0a6fc6e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3dad1f1d711b4450b1bf79ceed51bdbf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b61c735df6504099b3511e40bf05cde2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8900411d22ca44f6aa53900ea4fa85dd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7c6ee5b1fe174670b49a576000fe0394] to archive 2024-12-10T20:16:34,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:34,587 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/c730f0d968f749478c7e515ee6daba09 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/c730f0d968f749478c7e515ee6daba09 2024-12-10T20:16:34,587 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f363aee6d03841be985d37d364eb53f7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f363aee6d03841be985d37d364eb53f7 2024-12-10T20:16:34,587 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cb6d8a0fe2734826a321e95ac1193788 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cb6d8a0fe2734826a321e95ac1193788 2024-12-10T20:16:34,590 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/73fe05a11c124c95a493448b04b4ff72 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/73fe05a11c124c95a493448b04b4ff72 2024-12-10T20:16:34,590 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/dc653801d6bc4322b051ede096c0d62f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/dc653801d6bc4322b051ede096c0d62f 2024-12-10T20:16:34,590 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a75f65a66d346b8a31e162c863023f6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a75f65a66d346b8a31e162c863023f6 2024-12-10T20:16:34,597 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b623132421a0429fa5f029421b83caf9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b623132421a0429fa5f029421b83caf9 2024-12-10T20:16:34,597 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7ae3d44ddd0a4c4b94a5414be0aade9a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7ae3d44ddd0a4c4b94a5414be0aade9a 2024-12-10T20:16:34,598 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f1938b81c6ab47e8b34a82e3c8d4aa5f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f1938b81c6ab47e8b34a82e3c8d4aa5f 2024-12-10T20:16:34,598 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f392f6cad38a4faa9e0c9fb1d5180867 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/f392f6cad38a4faa9e0c9fb1d5180867 2024-12-10T20:16:34,599 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/57e7164e08394b9296689afcaf74ff64 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/57e7164e08394b9296689afcaf74ff64 2024-12-10T20:16:34,599 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4f812cdeae0a400c9e877c05a64d71e1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4f812cdeae0a400c9e877c05a64d71e1 2024-12-10T20:16:34,599 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3ee3dc5de0a041cfb52b6f033353ce9b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3ee3dc5de0a041cfb52b6f033353ce9b 2024-12-10T20:16:34,600 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9d45bc5b3b05494ba371580ab72c1c68 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9d45bc5b3b05494ba371580ab72c1c68 2024-12-10T20:16:34,600 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/205c5bcb4f1e4eb590b1f2f4871a847a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/205c5bcb4f1e4eb590b1f2f4871a847a 2024-12-10T20:16:34,601 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/93e59093f42d4c67b3b09e9672a6ee36 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/93e59093f42d4c67b3b09e9672a6ee36 2024-12-10T20:16:34,601 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4b4569c458fa4a9b9dd594a24846dffc to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4b4569c458fa4a9b9dd594a24846dffc 2024-12-10T20:16:34,601 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/23c0b6ed7cfa4ec88acb8c243d82ffd4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/23c0b6ed7cfa4ec88acb8c243d82ffd4 2024-12-10T20:16:34,602 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b379e486d03413db1424490cdcfeb72 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b379e486d03413db1424490cdcfeb72 2024-12-10T20:16:34,602 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/29723507b50d4f45bc37df60c5f14b7e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/29723507b50d4f45bc37df60c5f14b7e 2024-12-10T20:16:34,602 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b7badbebcfa439db6aea8d46f4eba56 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/1b7badbebcfa439db6aea8d46f4eba56 2024-12-10T20:16:34,602 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a19bafd5d504cd39243cc1d943c351c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/4a19bafd5d504cd39243cc1d943c351c 2024-12-10T20:16:34,603 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/269d374a2ca345b386ea248be3aba5b9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/269d374a2ca345b386ea248be3aba5b9 2024-12-10T20:16:34,603 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/d22ceefab8114bfdbecbcddaf3ae3ff2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/d22ceefab8114bfdbecbcddaf3ae3ff2 2024-12-10T20:16:34,603 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ea64870ca2644a1393855d9d8edadf67 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ea64870ca2644a1393855d9d8edadf67 2024-12-10T20:16:34,604 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/35bb4c27e5904ea8a08a3b3c88a6144d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/35bb4c27e5904ea8a08a3b3c88a6144d 2024-12-10T20:16:34,604 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/394fb1b75ed04c4eaf8aba2e409708f2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/394fb1b75ed04c4eaf8aba2e409708f2 2024-12-10T20:16:34,604 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9a0fde8275ab4d5ca865e867b84a8eb3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/9a0fde8275ab4d5ca865e867b84a8eb3 2024-12-10T20:16:34,604 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/74ff9ad9b1da4b7794617dc27746033b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/74ff9ad9b1da4b7794617dc27746033b 2024-12-10T20:16:34,605 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8e5c976475384946ad681314b0a6fc6e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8e5c976475384946ad681314b0a6fc6e 2024-12-10T20:16:34,605 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ba1c3b4c2f2546f49e9d5e609acea9d6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/ba1c3b4c2f2546f49e9d5e609acea9d6 2024-12-10T20:16:34,605 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b61c735df6504099b3511e40bf05cde2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/b61c735df6504099b3511e40bf05cde2 2024-12-10T20:16:34,605 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8900411d22ca44f6aa53900ea4fa85dd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/8900411d22ca44f6aa53900ea4fa85dd 2024-12-10T20:16:34,605 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7c6ee5b1fe174670b49a576000fe0394 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/7c6ee5b1fe174670b49a576000fe0394 2024-12-10T20:16:34,605 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3f0f070f94c14602a9d7afd41f34a336 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3f0f070f94c14602a9d7afd41f34a336 2024-12-10T20:16:34,606 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3dad1f1d711b4450b1bf79ceed51bdbf to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/3dad1f1d711b4450b1bf79ceed51bdbf 2024-12-10T20:16:34,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d572c60557f64e9ab67c735872727b2b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/192639d35e53466ca0d723dc7954b734, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6448ae8462f8403c82e6f4a0f7551ac4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d44741011557479495e7d2bae714ad53, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/8b5ec2253e34439a8bbf8d05cdff2e41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/58a90aa9544244ee9b7f74b93d942ed6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/85f6248f02534854b3134f44fad1bbdf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/509542d984bc41aea1eb56489e5ed139, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1394e4f572d34651a0319680fe3a5cc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0c7099991e0543c1b6e9a80424169323, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c6936b9add6f41b398e37a1e539e94ab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/388d835521c04a63bf39f8df3ad64b2c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ef2612579d134b7a958af47a9c3923d1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c10cd159140f452ca84967429d44fa08, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ee8eeeab555b431b918eab695f36dd90, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/887f421f0409417badbe94b26002e628, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/47177f97be5a4f56b19d26bc81b0883c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/32a93d09e8e442409477c39a0e5439d4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/644c2f63791c44ddb826985700de6437, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/4f77097a62434108a8e232bb9e338382, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a46c8fcb02a04eec935e95b1920f64e8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/5a7cf764bb294280899c56232fbbe3f6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1ed7aa5d3d1415fad03f4ba82c2d73b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/2258e28340dd445eb74b78b93d5bd480, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a955783c2dea414a9007a3dd41f0adbb, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ccc8d43dc1c8430f982da574fd3e7dbf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/698e590fe9624917ba473e1d45dfc07f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6ef08dbff5c44d2395d362387da215f4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7c783680760b4986b4b4989d0b22af61, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0ae2b66604aa455fb3a4cfee5fb582ea, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c5fccdf950bb4c6798409586c31fee49, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/392a6bfd27bc4a2ea139d04de6db5c1b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/51db4dcaf8b340458f37ad0d1fb9efb5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/229e96038bd04cc7bdc5830c4088faba, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7937e158c28b4a7a8387f726b392dae8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/35f0af8044a141dd872c31404a9aac76] to archive 2024-12-10T20:16:34,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:34,611 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/192639d35e53466ca0d723dc7954b734 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/192639d35e53466ca0d723dc7954b734 2024-12-10T20:16:34,611 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d44741011557479495e7d2bae714ad53 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d44741011557479495e7d2bae714ad53 2024-12-10T20:16:34,611 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6448ae8462f8403c82e6f4a0f7551ac4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6448ae8462f8403c82e6f4a0f7551ac4 2024-12-10T20:16:34,612 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d572c60557f64e9ab67c735872727b2b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d572c60557f64e9ab67c735872727b2b 2024-12-10T20:16:34,612 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/58a90aa9544244ee9b7f74b93d942ed6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/58a90aa9544244ee9b7f74b93d942ed6 2024-12-10T20:16:34,612 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/509542d984bc41aea1eb56489e5ed139 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/509542d984bc41aea1eb56489e5ed139 2024-12-10T20:16:34,613 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/85f6248f02534854b3134f44fad1bbdf to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/85f6248f02534854b3134f44fad1bbdf 2024-12-10T20:16:34,613 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/8b5ec2253e34439a8bbf8d05cdff2e41 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/8b5ec2253e34439a8bbf8d05cdff2e41 2024-12-10T20:16:34,613 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0c7099991e0543c1b6e9a80424169323 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0c7099991e0543c1b6e9a80424169323 2024-12-10T20:16:34,614 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c10cd159140f452ca84967429d44fa08 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c10cd159140f452ca84967429d44fa08 2024-12-10T20:16:34,614 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ef2612579d134b7a958af47a9c3923d1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ef2612579d134b7a958af47a9c3923d1 2024-12-10T20:16:34,615 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1394e4f572d34651a0319680fe3a5cc5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1394e4f572d34651a0319680fe3a5cc5 2024-12-10T20:16:34,620 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/388d835521c04a63bf39f8df3ad64b2c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/388d835521c04a63bf39f8df3ad64b2c 2024-12-10T20:16:34,620 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c6936b9add6f41b398e37a1e539e94ab to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c6936b9add6f41b398e37a1e539e94ab 2024-12-10T20:16:34,621 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ee8eeeab555b431b918eab695f36dd90 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ee8eeeab555b431b918eab695f36dd90 2024-12-10T20:16:34,621 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/47177f97be5a4f56b19d26bc81b0883c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/47177f97be5a4f56b19d26bc81b0883c 2024-12-10T20:16:34,622 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/32a93d09e8e442409477c39a0e5439d4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/32a93d09e8e442409477c39a0e5439d4 2024-12-10T20:16:34,622 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/644c2f63791c44ddb826985700de6437 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/644c2f63791c44ddb826985700de6437 2024-12-10T20:16:34,622 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/887f421f0409417badbe94b26002e628 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/887f421f0409417badbe94b26002e628 2024-12-10T20:16:34,622 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/5a7cf764bb294280899c56232fbbe3f6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/5a7cf764bb294280899c56232fbbe3f6 2024-12-10T20:16:34,623 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a46c8fcb02a04eec935e95b1920f64e8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a46c8fcb02a04eec935e95b1920f64e8 2024-12-10T20:16:34,625 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/2258e28340dd445eb74b78b93d5bd480 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/2258e28340dd445eb74b78b93d5bd480 2024-12-10T20:16:34,625 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1ed7aa5d3d1415fad03f4ba82c2d73b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1ed7aa5d3d1415fad03f4ba82c2d73b 2024-12-10T20:16:34,626 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a955783c2dea414a9007a3dd41f0adbb to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/a955783c2dea414a9007a3dd41f0adbb 2024-12-10T20:16:34,626 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/4f77097a62434108a8e232bb9e338382 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/4f77097a62434108a8e232bb9e338382 2024-12-10T20:16:34,626 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6ef08dbff5c44d2395d362387da215f4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/6ef08dbff5c44d2395d362387da215f4 2024-12-10T20:16:34,628 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/229e96038bd04cc7bdc5830c4088faba to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/229e96038bd04cc7bdc5830c4088faba 2024-12-10T20:16:34,628 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ccc8d43dc1c8430f982da574fd3e7dbf to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/ccc8d43dc1c8430f982da574fd3e7dbf 2024-12-10T20:16:34,628 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/392a6bfd27bc4a2ea139d04de6db5c1b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/392a6bfd27bc4a2ea139d04de6db5c1b 2024-12-10T20:16:34,628 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0ae2b66604aa455fb3a4cfee5fb582ea to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/0ae2b66604aa455fb3a4cfee5fb582ea 2024-12-10T20:16:34,628 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c5fccdf950bb4c6798409586c31fee49 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/c5fccdf950bb4c6798409586c31fee49 2024-12-10T20:16:34,629 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/35f0af8044a141dd872c31404a9aac76 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/35f0af8044a141dd872c31404a9aac76 2024-12-10T20:16:34,629 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7937e158c28b4a7a8387f726b392dae8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7937e158c28b4a7a8387f726b392dae8 2024-12-10T20:16:34,629 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/51db4dcaf8b340458f37ad0d1fb9efb5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/51db4dcaf8b340458f37ad0d1fb9efb5 2024-12-10T20:16:34,629 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/698e590fe9624917ba473e1d45dfc07f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/698e590fe9624917ba473e1d45dfc07f 2024-12-10T20:16:34,629 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7c783680760b4986b4b4989d0b22af61 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/7c783680760b4986b4b4989d0b22af61 2024-12-10T20:16:34,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/66e154f1147c4507ae03aa7f24947242, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ff5373c272334f0da27722834bbef14f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/824fffaf6c5b481f82d0afb04cc14c6e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/8b341a5735aa49739278c563fc8e0e06, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c8a67a494bbd49eb8093ac3b3bbb3d02, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7d22c2c7da874691ad725b9476eced59, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ac82f0475f404bd2a6e6ee432fa5199f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/4d4f1587a23240d48394be6a58774681, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/fdeabeabd30a4397ae369b4411e44e2f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6c502eddae8545e2b0ae507110e005ff, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/708b60f7e85e465894b1a108c0abe031, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/a791ba8933c04a36b6e7690211fe6d7a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c9909b3aa34a00a53316b760223fb0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7070f1b543074f0b9cf9663d90caf7b0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/be2633b28d52400abf488cd6985f0c9b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/5be250b3a9114d939db4f590ea948bce, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/f6e2c03699034a319b43916692697017, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/60379e909f584941b9c21bb57786bcde, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/875efa66327641faab9290d44ec5b9b7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38b541d568f44f5ba6d9fc7907cdc792, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e39c24e646a84aa8985b3d29e1eee7e2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38f8a6e708c647bbbef03f9079e62e1b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/576bed617d7e4ea2a6dd78803e867b51, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/cc751339f7084be8816df15047dc481f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ea792f3a3eab4371aac1f96f08bea4d7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c269efc4a74fa68836346fdd47d9b7, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6e89dc1137fb4118b3ddfd3dd3b5ea41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e384ef94c17e472aa8390d7e06dcabb8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e64d72ed71144a969128ad8951898cc5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9aacc4c51d7c4593a1ff2a87796f31fb, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9083f10774394682b52d2347b636d63c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/26a88e6ef2c942d08e4518e66d7293ec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/1f804e7260524be5bf7c955d5df09dbd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/2b8e555af50c466f85ef2180bf431309, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/65eea3238ca64bfd96a8ca1fe6d2c4ad, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6b1ac66474ed4b938ffaa30aa8fb51e7] to archive 2024-12-10T20:16:34,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:34,640 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/66e154f1147c4507ae03aa7f24947242 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/66e154f1147c4507ae03aa7f24947242 2024-12-10T20:16:34,648 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/8b341a5735aa49739278c563fc8e0e06 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/8b341a5735aa49739278c563fc8e0e06 2024-12-10T20:16:34,648 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/824fffaf6c5b481f82d0afb04cc14c6e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/824fffaf6c5b481f82d0afb04cc14c6e 2024-12-10T20:16:34,648 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7d22c2c7da874691ad725b9476eced59 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7d22c2c7da874691ad725b9476eced59 2024-12-10T20:16:34,649 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/fdeabeabd30a4397ae369b4411e44e2f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/fdeabeabd30a4397ae369b4411e44e2f 2024-12-10T20:16:34,649 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ac82f0475f404bd2a6e6ee432fa5199f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ac82f0475f404bd2a6e6ee432fa5199f 2024-12-10T20:16:34,650 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c8a67a494bbd49eb8093ac3b3bbb3d02 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c8a67a494bbd49eb8093ac3b3bbb3d02 2024-12-10T20:16:34,650 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7070f1b543074f0b9cf9663d90caf7b0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/7070f1b543074f0b9cf9663d90caf7b0 2024-12-10T20:16:34,652 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6c502eddae8545e2b0ae507110e005ff to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6c502eddae8545e2b0ae507110e005ff 2024-12-10T20:16:34,653 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c9909b3aa34a00a53316b760223fb0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c9909b3aa34a00a53316b760223fb0 2024-12-10T20:16:34,653 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/be2633b28d52400abf488cd6985f0c9b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/be2633b28d52400abf488cd6985f0c9b 2024-12-10T20:16:34,654 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/f6e2c03699034a319b43916692697017 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/f6e2c03699034a319b43916692697017 2024-12-10T20:16:34,655 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/875efa66327641faab9290d44ec5b9b7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/875efa66327641faab9290d44ec5b9b7 2024-12-10T20:16:34,655 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/5be250b3a9114d939db4f590ea948bce to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/5be250b3a9114d939db4f590ea948bce 2024-12-10T20:16:34,655 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/60379e909f584941b9c21bb57786bcde to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/60379e909f584941b9c21bb57786bcde 2024-12-10T20:16:34,656 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ff5373c272334f0da27722834bbef14f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ff5373c272334f0da27722834bbef14f 2024-12-10T20:16:34,656 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/708b60f7e85e465894b1a108c0abe031 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/708b60f7e85e465894b1a108c0abe031 2024-12-10T20:16:34,656 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/4d4f1587a23240d48394be6a58774681 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/4d4f1587a23240d48394be6a58774681 2024-12-10T20:16:34,657 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e39c24e646a84aa8985b3d29e1eee7e2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e39c24e646a84aa8985b3d29e1eee7e2 2024-12-10T20:16:34,657 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38b541d568f44f5ba6d9fc7907cdc792 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38b541d568f44f5ba6d9fc7907cdc792 2024-12-10T20:16:34,658 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/cc751339f7084be8816df15047dc481f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/cc751339f7084be8816df15047dc481f 2024-12-10T20:16:34,658 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/576bed617d7e4ea2a6dd78803e867b51 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/576bed617d7e4ea2a6dd78803e867b51 2024-12-10T20:16:34,659 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38f8a6e708c647bbbef03f9079e62e1b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/38f8a6e708c647bbbef03f9079e62e1b 2024-12-10T20:16:34,659 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/a791ba8933c04a36b6e7690211fe6d7a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/a791ba8933c04a36b6e7690211fe6d7a 2024-12-10T20:16:34,660 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c269efc4a74fa68836346fdd47d9b7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/c2c269efc4a74fa68836346fdd47d9b7 2024-12-10T20:16:34,661 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6e89dc1137fb4118b3ddfd3dd3b5ea41 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6e89dc1137fb4118b3ddfd3dd3b5ea41 2024-12-10T20:16:34,661 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e64d72ed71144a969128ad8951898cc5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e64d72ed71144a969128ad8951898cc5 2024-12-10T20:16:34,661 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e384ef94c17e472aa8390d7e06dcabb8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/e384ef94c17e472aa8390d7e06dcabb8 2024-12-10T20:16:34,665 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/26a88e6ef2c942d08e4518e66d7293ec to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/26a88e6ef2c942d08e4518e66d7293ec 2024-12-10T20:16:34,665 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/65eea3238ca64bfd96a8ca1fe6d2c4ad to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/65eea3238ca64bfd96a8ca1fe6d2c4ad 2024-12-10T20:16:34,665 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/2b8e555af50c466f85ef2180bf431309 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/2b8e555af50c466f85ef2180bf431309 2024-12-10T20:16:34,665 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6b1ac66474ed4b938ffaa30aa8fb51e7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/6b1ac66474ed4b938ffaa30aa8fb51e7 2024-12-10T20:16:34,667 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9083f10774394682b52d2347b636d63c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9083f10774394682b52d2347b636d63c 2024-12-10T20:16:34,668 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/1f804e7260524be5bf7c955d5df09dbd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/1f804e7260524be5bf7c955d5df09dbd 2024-12-10T20:16:34,670 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9aacc4c51d7c4593a1ff2a87796f31fb to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/9aacc4c51d7c4593a1ff2a87796f31fb 2024-12-10T20:16:34,670 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ea792f3a3eab4371aac1f96f08bea4d7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/ea792f3a3eab4371aac1f96f08bea4d7 2024-12-10T20:16:34,676 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/recovered.edits/545.seqid, newMaxSeqId=545, maxSeqId=1 2024-12-10T20:16:34,676 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. 2024-12-10T20:16:34,676 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for d574f3055e0a5aa98dce8c43f5c73a8d: Waiting for close lock at 1733861793327Running coprocessor pre-close hooks at 1733861793327Disabling compacts and flushes for region at 1733861793327Disabling writes for close at 1733861793327Obtaining lock to block concurrent updates at 1733861793327Preparing flush snapshotting stores in d574f3055e0a5aa98dce8c43f5c73a8d at 1733861793327Finished memstore snapshotting TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733861793328 (+1 ms)Flushing stores of TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d. at 1733861793328Flushing d574f3055e0a5aa98dce8c43f5c73a8d/A: creating writer at 1733861793328Flushing d574f3055e0a5aa98dce8c43f5c73a8d/A: appending metadata at 1733861793330 (+2 ms)Flushing d574f3055e0a5aa98dce8c43f5c73a8d/A: closing flushed file at 1733861793330Flushing d574f3055e0a5aa98dce8c43f5c73a8d/B: creating writer at 1733861793738 (+408 ms)Flushing d574f3055e0a5aa98dce8c43f5c73a8d/B: appending metadata at 1733861793740 (+2 ms)Flushing d574f3055e0a5aa98dce8c43f5c73a8d/B: closing flushed file at 1733861793740Flushing d574f3055e0a5aa98dce8c43f5c73a8d/C: creating writer at 1733861794151 (+411 ms)Flushing d574f3055e0a5aa98dce8c43f5c73a8d/C: appending metadata at 1733861794154 (+3 ms)Flushing d574f3055e0a5aa98dce8c43f5c73a8d/C: closing flushed file at 1733861794154Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ddaeaa: reopening flushed file at 1733861794563 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@262956b1: reopening flushed file at 1733861794568 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42a961ee: reopening flushed file at 1733861794575 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for d574f3055e0a5aa98dce8c43f5c73a8d in 1254ms, sequenceid=542, compaction requested=false at 1733861794581 (+6 ms)Writing region close event to WAL at 1733861794673 (+92 ms)Running coprocessor post-close hooks at 1733861794676 (+3 ms)Closed at 1733861794676 2024-12-10T20:16:34,681 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:34,687 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=d574f3055e0a5aa98dce8c43f5c73a8d, regionState=CLOSED 2024-12-10T20:16:34,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:16:34,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-10T20:16:34,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure d574f3055e0a5aa98dce8c43f5c73a8d, server=03a2ae042c95,38133,1733861689839 in 1.5170 sec 2024-12-10T20:16:34,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-10T20:16:34,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d574f3055e0a5aa98dce8c43f5c73a8d, UNASSIGN in 1.5240 sec 2024-12-10T20:16:34,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-10T20:16:34,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5280 sec 2024-12-10T20:16:34,699 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861794699"}]},"ts":"1733861794699"} 2024-12-10T20:16:34,700 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T20:16:34,700 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T20:16:34,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5400 sec 2024-12-10T20:16:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T20:16:35,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:35,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T20:16:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,310 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T20:16:35,310 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,312 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:35,314 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/recovered.edits] 2024-12-10T20:16:35,317 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/708ba96d53ea48d69d069e2e7bdb6485 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/708ba96d53ea48d69d069e2e7bdb6485 2024-12-10T20:16:35,317 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cc0c8dcdcfd14adfa0131d9940396731 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/A/cc0c8dcdcfd14adfa0131d9940396731 2024-12-10T20:16:35,319 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1e795929a0304b658e4deaa6cd8d10ad to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/1e795929a0304b658e4deaa6cd8d10ad 2024-12-10T20:16:35,319 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1d7263f4be14120910ccc5b3327e3a1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/B/d1d7263f4be14120910ccc5b3327e3a1 2024-12-10T20:16:35,321 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/83e5ac0dec364442a5327d8f934d96dd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/83e5ac0dec364442a5327d8f934d96dd 2024-12-10T20:16:35,321 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/3eea98e654a049bb9efb1f3739b1075f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/C/3eea98e654a049bb9efb1f3739b1075f 2024-12-10T20:16:35,323 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/recovered.edits/545.seqid to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d/recovered.edits/545.seqid 2024-12-10T20:16:35,324 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/d574f3055e0a5aa98dce8c43f5c73a8d 2024-12-10T20:16:35,324 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T20:16:35,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,328 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T20:16:35,330 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T20:16:35,332 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,332 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T20:16:35,332 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733861795332"}]},"ts":"9223372036854775807"} 2024-12-10T20:16:35,334 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T20:16:35,334 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d574f3055e0a5aa98dce8c43f5c73a8d, NAME => 'TestAcidGuarantees,,1733861761749.d574f3055e0a5aa98dce8c43f5c73a8d.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T20:16:35,334 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T20:16:35,335 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733861795334"}]},"ts":"9223372036854775807"} 2024-12-10T20:16:35,336 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T20:16:35,337 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-12-10T20:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T20:16:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T20:16:35,421 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:35,429 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251 (was 253), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=798 (was 848), ProcessCount=11 (was 11), AvailableMemoryMB=1337 (was 867) - AvailableMemoryMB LEAK? - 2024-12-10T20:16:35,439 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=798, ProcessCount=11, AvailableMemoryMB=1337 2024-12-10T20:16:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:16:35,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:16:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:35,443 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:16:35,443 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:35,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-12-10T20:16:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T20:16:35,444 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:16:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742179_1355 (size=1061) 2024-12-10T20:16:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T20:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T20:16:35,850 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:16:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742180_1356 (size=53) 2024-12-10T20:16:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T20:16:36,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:16:36,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 85784434655d82701a348d7dd2302a0e, disabling compactions & flushes 2024-12-10T20:16:36,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. after waiting 0 ms 2024-12-10T20:16:36,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,257 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 85784434655d82701a348d7dd2302a0e: Waiting for close lock at 1733861796256Disabling compacts and flushes for region at 1733861796256Disabling writes for close at 1733861796256Writing region close event to WAL at 1733861796256Closed at 1733861796256 2024-12-10T20:16:36,258 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:16:36,259 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733861796258"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733861796258"}]},"ts":"1733861796258"} 2024-12-10T20:16:36,261 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:16:36,263 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:16:36,263 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861796263"}]},"ts":"1733861796263"} 2024-12-10T20:16:36,264 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T20:16:36,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, ASSIGN}] 2024-12-10T20:16:36,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, ASSIGN 2024-12-10T20:16:36,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, ASSIGN; state=OFFLINE, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=false 2024-12-10T20:16:36,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:36,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, ASSIGN because future has completed 2024-12-10T20:16:36,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:16:36,577 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T20:16:36,578 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:16:36,578 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,578 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:16:36,578 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,579 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,581 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,583 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:36,583 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85784434655d82701a348d7dd2302a0e columnFamilyName A 2024-12-10T20:16:36,583 DEBUG [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:36,584 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(327): Store=85784434655d82701a348d7dd2302a0e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:36,584 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,585 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:36,585 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85784434655d82701a348d7dd2302a0e columnFamilyName B 2024-12-10T20:16:36,585 DEBUG [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:36,586 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(327): Store=85784434655d82701a348d7dd2302a0e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:36,586 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,587 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:36,587 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85784434655d82701a348d7dd2302a0e columnFamilyName C 2024-12-10T20:16:36,587 DEBUG [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:36,587 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(327): Store=85784434655d82701a348d7dd2302a0e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:36,587 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,587 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,588 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,588 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,589 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,589 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,590 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:16:36,591 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,592 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:16:36,592 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 85784434655d82701a348d7dd2302a0e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70349773, jitterRate=0.048293307423591614}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:16:36,592 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:36,593 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 85784434655d82701a348d7dd2302a0e: Running coprocessor pre-open hook at 1733861796579Writing region info on filesystem at 1733861796579Initializing all the Stores at 1733861796580 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861796580Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861796580Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861796580Cleaning up temporary data from old regions at 1733861796589 (+9 ms)Running coprocessor post-open hooks at 1733861796592 (+3 ms)Region opened successfully at 1733861796593 (+1 ms) 2024-12-10T20:16:36,594 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., pid=83, masterSystemTime=1733861796573 2024-12-10T20:16:36,595 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,595 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:36,595 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:36,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:16:36,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-10T20:16:36,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 in 176 msec 2024-12-10T20:16:36,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-10T20:16:36,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, ASSIGN in 333 msec 2024-12-10T20:16:36,600 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:16:36,600 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861796600"}]},"ts":"1733861796600"} 2024-12-10T20:16:36,601 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T20:16:36,602 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:16:36,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-12-10T20:16:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-10T20:16:37,587 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:37,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:37,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:37,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:37,589 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51fbb8a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:37,591 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb5ac36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:37,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:37,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:37,595 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:16:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:16:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b856a41 2024-12-10T20:16:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:16:37,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:16:37,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:16:37,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:16:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T20:16:37,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742181_1357 (size=1097) 2024-12-10T20:16:38,013 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T20:16:38,013 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T20:16:38,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:16:38,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, REOPEN/MOVE}] 2024-12-10T20:16:38,018 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, REOPEN/MOVE 2024-12-10T20:16:38,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, REOPEN/MOVE because future has completed 2024-12-10T20:16:38,020 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:16:38,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:16:38,173 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,173 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:16:38,173 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 85784434655d82701a348d7dd2302a0e, disabling compactions & flushes 2024-12-10T20:16:38,173 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,174 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,174 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. after waiting 0 ms 2024-12-10T20:16:38,174 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,179 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T20:16:38,181 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,181 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 85784434655d82701a348d7dd2302a0e: Waiting for close lock at 1733861798173Running coprocessor pre-close hooks at 1733861798173Disabling compacts and flushes for region at 1733861798173Disabling writes for close at 1733861798174 (+1 ms)Writing region close event to WAL at 1733861798176 (+2 ms)Running coprocessor post-close hooks at 1733861798180 (+4 ms)Closed at 1733861798181 (+1 ms) 2024-12-10T20:16:38,181 WARN [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 85784434655d82701a348d7dd2302a0e to self. 2024-12-10T20:16:38,183 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,184 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=CLOSED 2024-12-10T20:16:38,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:16:38,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-10T20:16:38,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 in 167 msec 2024-12-10T20:16:38,190 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, REOPEN/MOVE; state=CLOSED, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=true 2024-12-10T20:16:38,341 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, REOPEN/MOVE because future has completed 2024-12-10T20:16:38,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:16:38,498 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,498 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:16:38,499 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,499 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:16:38,499 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,499 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,500 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,501 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:38,501 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85784434655d82701a348d7dd2302a0e columnFamilyName A 2024-12-10T20:16:38,502 DEBUG [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:38,503 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(327): Store=85784434655d82701a348d7dd2302a0e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:38,503 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,504 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:38,504 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85784434655d82701a348d7dd2302a0e columnFamilyName B 2024-12-10T20:16:38,504 DEBUG [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:38,505 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(327): Store=85784434655d82701a348d7dd2302a0e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:38,505 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,506 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:16:38,506 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85784434655d82701a348d7dd2302a0e columnFamilyName C 2024-12-10T20:16:38,506 DEBUG [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:38,507 INFO [StoreOpener-85784434655d82701a348d7dd2302a0e-1 {}] regionserver.HStore(327): Store=85784434655d82701a348d7dd2302a0e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:16:38,507 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,507 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,507 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,508 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,509 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,510 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,510 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:16:38,512 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,513 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 85784434655d82701a348d7dd2302a0e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59448544, jitterRate=-0.11414766311645508}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:16:38,513 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,514 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 85784434655d82701a348d7dd2302a0e: Running coprocessor pre-open hook at 1733861798499Writing region info on filesystem at 1733861798499Initializing all the Stores at 1733861798499Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861798500 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861798500Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861798500Cleaning up temporary data from old regions at 1733861798510 (+10 ms)Running coprocessor post-open hooks at 1733861798513 (+3 ms)Region opened successfully at 1733861798513 2024-12-10T20:16:38,515 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., pid=88, masterSystemTime=1733861798495 2024-12-10T20:16:38,517 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,517 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,517 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=OPEN, openSeqNum=5, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:16:38,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-12-10T20:16:38,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 in 177 msec 2024-12-10T20:16:38,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-10T20:16:38,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, REOPEN/MOVE in 505 msec 2024-12-10T20:16:38,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-10T20:16:38,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-12-10T20:16:38,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-12-10T20:16:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T20:16:38,533 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:38,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7288b4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,534 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081f862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d7f1de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69edc398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,539 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214f6b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,542 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc57482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61351bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4c965c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60780b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6444e4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,550 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbf8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,552 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1740426e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b350ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,555 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63fa4491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304607c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ee060c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,566 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267de23a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37172362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45749f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486039d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b9c6bff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a19ac61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7731bda1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,589 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c01315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1657d9e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5bb9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fc8261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adc9e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:16:38,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:16:38,597 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:16:38,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:16:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:16:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d83d6a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:16:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:16:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:38,602 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:16:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4e8ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:16:38,609 DEBUG [Thread-1628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:16:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:38,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:38,629 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,629 DEBUG [Thread-1632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,629 DEBUG [Thread-1637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,633 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,649 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,650 DEBUG [Thread-1624 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,650 DEBUG [Thread-1624 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,653 DEBUG [Thread-1626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,669 DEBUG [Thread-1630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:38,671 DEBUG [Thread-1639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-10T20:16:38,675 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:38,676 DEBUG [Thread-1641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,676 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:38,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:38,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,677 DEBUG [Thread-1641 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,677 DEBUG [Thread-1641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861798650, max timestamp=1733861798671Num uniques 859; 2024-12-10T20:16:38,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861798664, max timestamp=1733861798671Num uniques 859; 2024-12-10T20:16:38,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:16:38,685 DEBUG [Thread-1643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:16:38,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:16:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:16:38,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861798664, max timestamp=1733861798671Num uniques 859; 2024-12-10T20:16:38,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:38,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:16:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:16:38,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103ed76eabfe15456e95853e0889c4b19d_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861798618/Put/seqid=0 2024-12-10T20:16:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861858738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861858738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861858740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861858740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861858742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742182_1358 (size=9714) 2024-12-10T20:16:38,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:38,791 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103ed76eabfe15456e95853e0889c4b19d_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103ed76eabfe15456e95853e0889c4b19d_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:38,792 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/3320f7c91be7416fbaa82b1510577a68, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:38,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/3320f7c91be7416fbaa82b1510577a68 is 175, key is test_row_0/A:col10/1733861798618/Put/seqid=0 2024-12-10T20:16:38,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:38,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742183_1359 (size=22361) 2024-12-10T20:16:38,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/3320f7c91be7416fbaa82b1510577a68 2024-12-10T20:16:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861858857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861858858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861858867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861858868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861858868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:38,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/06595f36c7ac4bbfa4c3d4540b73db47 is 50, key is test_row_0/B:col10/1733861798618/Put/seqid=0 2024-12-10T20:16:38,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742184_1360 (size=9657) 2024-12-10T20:16:38,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/06595f36c7ac4bbfa4c3d4540b73db47 2024-12-10T20:16:38,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/0e23ea73d39d44d59a93079222df9c09 is 50, key is test_row_0/C:col10/1733861798618/Put/seqid=0 2024-12-10T20:16:38,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742185_1361 (size=9657) 2024-12-10T20:16:38,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:38,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:38,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861859078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861859078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861859079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861859088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861859088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:39,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/0e23ea73d39d44d59a93079222df9c09 2024-12-10T20:16:39,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/3320f7c91be7416fbaa82b1510577a68 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68 2024-12-10T20:16:39,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68, entries=100, sequenceid=14, filesize=21.8 K 2024-12-10T20:16:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/06595f36c7ac4bbfa4c3d4540b73db47 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/06595f36c7ac4bbfa4c3d4540b73db47 2024-12-10T20:16:39,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/06595f36c7ac4bbfa4c3d4540b73db47, entries=100, sequenceid=14, filesize=9.4 K 2024-12-10T20:16:39,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861859387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/0e23ea73d39d44d59a93079222df9c09 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0e23ea73d39d44d59a93079222df9c09 2024-12-10T20:16:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861859387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861859387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0e23ea73d39d44d59a93079222df9c09, entries=100, sequenceid=14, filesize=9.4 K 2024-12-10T20:16:39,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 85784434655d82701a348d7dd2302a0e in 778ms, sequenceid=14, compaction requested=false 2024-12-10T20:16:39,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:39,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861798711, max timestamp=1733861799406Num uniques 1118; 2024-12-10T20:16:39,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-10T20:16:39,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861798711, max timestamp=1733861799406Num uniques 1018; 2024-12-10T20:16:39,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T20:16:39,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733861798711, max timestamp=1733861799406Num uniques 1018; 2024-12-10T20:16:39,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-10T20:16:39,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=46.16 KB heapSize=123.33 KB 2024-12-10T20:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:39,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:39,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100e29f195d20841d9bdd64eba470a80b2_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861798738/Put/seqid=0 2024-12-10T20:16:39,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742186_1362 (size=17034) 2024-12-10T20:16:39,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:39,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100e29f195d20841d9bdd64eba470a80b2_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100e29f195d20841d9bdd64eba470a80b2_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:39,496 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/342744ced1ab44519a555e88c58349be, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:39,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/342744ced1ab44519a555e88c58349be is 175, key is test_row_0/A:col10/1733861798738/Put/seqid=0 2024-12-10T20:16:39,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861799486, max timestamp=1733861799528Num uniques 859; 2024-12-10T20:16:39,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861799486, max timestamp=1733861799528Num uniques 859; 2024-12-10T20:16:39,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:39,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:39,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861799486, max timestamp=1733861799528Num uniques 859; 2024-12-10T20:16:39,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:16:39,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742187_1363 (size=48139) 2024-12-10T20:16:39,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861799605, max timestamp=1733861799635Num uniques 1018; 2024-12-10T20:16:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T20:16:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861799605, max timestamp=1733861799635Num uniques 1018; 2024-12-10T20:16:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T20:16:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733861799605, max timestamp=1733861799635Num uniques 1018; 2024-12-10T20:16:39,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T20:16:39,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1733861799704, max timestamp=1733861799749Num uniques 1027; 2024-12-10T20:16:39,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47022, new segment data size=8092, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T20:16:39,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:39,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:39,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733861799704, max timestamp=1733861799749Num uniques 1027; 2024-12-10T20:16:39,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47027, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-12-10T20:16:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733861799704, max timestamp=1733861799749Num uniques 1027; 2024-12-10T20:16:39,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47027, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-12-10T20:16:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:39,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733861799853, max timestamp=1733861799889Num uniques 1036; 2024-12-10T20:16:39,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-10T20:16:39,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733861799853, max timestamp=1733861799889Num uniques 1009; 2024-12-10T20:16:39,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46219, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-12-10T20:16:39,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733861799853, max timestamp=1733861799889Num uniques 1009; 2024-12-10T20:16:39,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46219, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-12-10T20:16:39,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:39,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:39,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/342744ced1ab44519a555e88c58349be 2024-12-10T20:16:39,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/5ec623328bc1498dad1b02c5b59ea3de is 50, key is test_row_0/B:col10/1733861798738/Put/seqid=0 2024-12-10T20:16:39,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742188_1364 (size=12001) 2024-12-10T20:16:39,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/5ec623328bc1498dad1b02c5b59ea3de 2024-12-10T20:16:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861859946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861859949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861859948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861859967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861859967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:39,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/66afedf28b4b474bb6cda898090ee433 is 50, key is test_row_0/C:col10/1733861798738/Put/seqid=0 2024-12-10T20:16:40,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742189_1365 (size=12001) 2024-12-10T20:16:40,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.86 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/66afedf28b4b474bb6cda898090ee433 2024-12-10T20:16:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/342744ced1ab44519a555e88c58349be as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be 2024-12-10T20:16:40,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be, entries=250, sequenceid=55, filesize=47.0 K 2024-12-10T20:16:40,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/5ec623328bc1498dad1b02c5b59ea3de as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/5ec623328bc1498dad1b02c5b59ea3de 2024-12-10T20:16:40,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/5ec623328bc1498dad1b02c5b59ea3de, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T20:16:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/66afedf28b4b474bb6cda898090ee433 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/66afedf28b4b474bb6cda898090ee433 2024-12-10T20:16:40,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/66afedf28b4b474bb6cda898090ee433, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T20:16:40,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.16 KB/47272, heapSize ~123.28 KB/126240, currentSize=140.89 KB/144270 for 85784434655d82701a348d7dd2302a0e in 621ms, sequenceid=55, compaction requested=false 2024-12-10T20:16:40,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:40,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-10T20:16:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:40,078 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=140.89 KB heapSize=397.69 KB 2024-12-10T20:16:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:40,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fd92be41865141abbfaa7688f97f9a8b_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861799959/Put/seqid=0 2024-12-10T20:16:40,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861860094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861860100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742190_1366 (size=12304) 2024-12-10T20:16:40,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861860102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861860105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861860105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,215 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:16:40,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861860217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861860217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861860227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861860227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861860227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861860440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861860440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861860447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861860447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861860447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:40,512 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fd92be41865141abbfaa7688f97f9a8b_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fd92be41865141abbfaa7688f97f9a8b_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/f8afbaa04d41449aac89252999d19641, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/f8afbaa04d41449aac89252999d19641 is 175, key is test_row_0/A:col10/1733861799959/Put/seqid=0 2024-12-10T20:16:40,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742191_1367 (size=31105) 2024-12-10T20:16:40,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861860757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861860757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861860766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861860767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861860767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:40,921 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/f8afbaa04d41449aac89252999d19641 2024-12-10T20:16:40,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/a82fa46223fa4dc2bd1e186215f14678 is 50, key is test_row_0/B:col10/1733861799959/Put/seqid=0 2024-12-10T20:16:40,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742192_1368 (size=12151) 2024-12-10T20:16:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861861278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861861278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:41,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861861287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:41,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861861287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:41,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861861287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:41,357 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/a82fa46223fa4dc2bd1e186215f14678 2024-12-10T20:16:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/d5d7a168bd354b6badafeab29e36b5f5 is 50, key is test_row_0/C:col10/1733861799959/Put/seqid=0 2024-12-10T20:16:41,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742193_1369 (size=12151) 2024-12-10T20:16:41,381 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/d5d7a168bd354b6badafeab29e36b5f5 2024-12-10T20:16:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/f8afbaa04d41449aac89252999d19641 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641 2024-12-10T20:16:41,400 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641, entries=150, sequenceid=143, filesize=30.4 K 2024-12-10T20:16:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/a82fa46223fa4dc2bd1e186215f14678 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/a82fa46223fa4dc2bd1e186215f14678 2024-12-10T20:16:41,422 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/a82fa46223fa4dc2bd1e186215f14678, entries=150, sequenceid=143, filesize=11.9 K 2024-12-10T20:16:41,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/d5d7a168bd354b6badafeab29e36b5f5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d5d7a168bd354b6badafeab29e36b5f5 2024-12-10T20:16:41,432 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d5d7a168bd354b6badafeab29e36b5f5, entries=150, sequenceid=143, filesize=11.9 K 2024-12-10T20:16:41,437 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~397.64 KB/407184, currentSize=46.96 KB/48090 for 85784434655d82701a348d7dd2302a0e in 1359ms, sequenceid=143, compaction requested=true 2024-12-10T20:16:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-10T20:16:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-10T20:16:41,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-10T20:16:41,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7630 sec 2024-12-10T20:16:41,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.7710 sec 2024-12-10T20:16:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:42,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:16:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:42,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:42,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121032ad348415994f09bcf8c06e1f4e9690_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:42,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742194_1370 (size=14794) 2024-12-10T20:16:42,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733861802316, max timestamp=1733861802342Num uniques 859; 2024-12-10T20:16:42,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733861802316, max timestamp=1733861802342Num uniques 859; 2024-12-10T20:16:42,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733861802316, max timestamp=1733861802342Num uniques 859; 2024-12-10T20:16:42,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T20:16:42,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T20:16:42,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T20:16:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861862361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861862365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861862365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861862366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861862367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861862476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861862477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861862487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861862487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861862487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861862811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861862812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861862812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861862811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861862812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:42,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121032ad348415994f09bcf8c06e1f4e9690_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121032ad348415994f09bcf8c06e1f4e9690_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:42,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/8baa94f095254051b6355ddd2365a9e0, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:42,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/8baa94f095254051b6355ddd2365a9e0 is 175, key is test_row_0/A:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:42,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742195_1371 (size=39749) 2024-12-10T20:16:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T20:16:42,837 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:42,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-10T20:16:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:42,989 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:42,990 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:42,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861863116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861863116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861863117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861863117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861863117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T20:16:43,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:43,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/8baa94f095254051b6355ddd2365a9e0 2024-12-10T20:16:43,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/d6ca1f6f812d4143871bd553e59b3027 is 50, key is test_row_0/B:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:43,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742196_1372 (size=12151) 2024-12-10T20:16:43,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T20:16:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:43,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T20:16:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T20:16:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:43,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/d6ca1f6f812d4143871bd553e59b3027 2024-12-10T20:16:43,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861863637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861863637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861863637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861863637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861863637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:43,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/8b91da2447fe4e60859108387ada97df is 50, key is test_row_0/C:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:43,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742197_1373 (size=12151) 2024-12-10T20:16:43,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/8b91da2447fe4e60859108387ada97df 2024-12-10T20:16:43,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/8baa94f095254051b6355ddd2365a9e0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0 2024-12-10T20:16:43,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0, entries=200, sequenceid=154, filesize=38.8 K 2024-12-10T20:16:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/d6ca1f6f812d4143871bd553e59b3027 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d6ca1f6f812d4143871bd553e59b3027 2024-12-10T20:16:43,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d6ca1f6f812d4143871bd553e59b3027, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T20:16:43,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/8b91da2447fe4e60859108387ada97df as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b91da2447fe4e60859108387ada97df 2024-12-10T20:16:43,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b91da2447fe4e60859108387ada97df, entries=150, sequenceid=154, filesize=11.9 K 2024-12-10T20:16:43,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 85784434655d82701a348d7dd2302a0e in 1370ms, sequenceid=154, compaction requested=true 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:43,673 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:43,673 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:43,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:43,675 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:16:43,675 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141354 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:16:43,675 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/B is initiating minor compaction (all files) 2024-12-10T20:16:43,675 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/A is initiating minor compaction (all files) 2024-12-10T20:16:43,675 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/A in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,675 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/B in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,676 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/06595f36c7ac4bbfa4c3d4540b73db47, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/5ec623328bc1498dad1b02c5b59ea3de, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/a82fa46223fa4dc2bd1e186215f14678, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d6ca1f6f812d4143871bd553e59b3027] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=44.9 K 2024-12-10T20:16:43,676 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=138.0 K 2024-12-10T20:16:43,676 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0] 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 06595f36c7ac4bbfa4c3d4540b73db47, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733861798618 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3320f7c91be7416fbaa82b1510577a68, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733861798618 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec623328bc1498dad1b02c5b59ea3de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733861798738 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 342744ced1ab44519a555e88c58349be, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733861798738 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8afbaa04d41449aac89252999d19641, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733861799947 2024-12-10T20:16:43,676 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a82fa46223fa4dc2bd1e186215f14678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733861799947 2024-12-10T20:16:43,677 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ca1f6f812d4143871bd553e59b3027, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733861800086 2024-12-10T20:16:43,677 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8baa94f095254051b6355ddd2365a9e0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733861800086 2024-12-10T20:16:43,681 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:43,683 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:43,684 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/fa01acc44fab499d8fc5982508f69b60 is 50, key is test_row_0/B:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:43,684 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210b27912709078432ea3e8a61d42afb581_85784434655d82701a348d7dd2302a0e store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:43,690 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210b27912709078432ea3e8a61d42afb581_85784434655d82701a348d7dd2302a0e, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:43,690 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b27912709078432ea3e8a61d42afb581_85784434655d82701a348d7dd2302a0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:43,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742198_1374 (size=12289) 2024-12-10T20:16:43,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742199_1375 (size=4469) 2024-12-10T20:16:43,703 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#A#compaction#322 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:43,704 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/bd9224ec82f54ceaa355c2243a47438f is 175, key is test_row_0/A:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:43,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742200_1376 (size=31243) 2024-12-10T20:16:43,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T20:16:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:43,753 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-10T20:16:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:43,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:43,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121039ba1ba33c0149b28e08906b5d65d81a_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861802365/Put/seqid=0 2024-12-10T20:16:43,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742201_1377 (size=12304) 2024-12-10T20:16:44,101 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/fa01acc44fab499d8fc5982508f69b60 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/fa01acc44fab499d8fc5982508f69b60 2024-12-10T20:16:44,105 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 85784434655d82701a348d7dd2302a0e/B of 85784434655d82701a348d7dd2302a0e into fa01acc44fab499d8fc5982508f69b60(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:44,105 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:44,105 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/B, priority=12, startTime=1733861803673; duration=0sec 2024-12-10T20:16:44,105 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:44,105 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:B 2024-12-10T20:16:44,105 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:16:44,106 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:16:44,106 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/C is initiating minor compaction (all files) 2024-12-10T20:16:44,106 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/C in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:44,106 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0e23ea73d39d44d59a93079222df9c09, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/66afedf28b4b474bb6cda898090ee433, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d5d7a168bd354b6badafeab29e36b5f5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b91da2447fe4e60859108387ada97df] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=44.9 K 2024-12-10T20:16:44,107 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e23ea73d39d44d59a93079222df9c09, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733861798618 2024-12-10T20:16:44,107 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 66afedf28b4b474bb6cda898090ee433, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733861798738 2024-12-10T20:16:44,107 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d7a168bd354b6badafeab29e36b5f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733861799947 2024-12-10T20:16:44,107 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b91da2447fe4e60859108387ada97df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733861800086 2024-12-10T20:16:44,111 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/bd9224ec82f54ceaa355c2243a47438f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/bd9224ec82f54ceaa355c2243a47438f 2024-12-10T20:16:44,115 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:44,115 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 85784434655d82701a348d7dd2302a0e/A of 85784434655d82701a348d7dd2302a0e into bd9224ec82f54ceaa355c2243a47438f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:44,115 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:44,115 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/A, priority=12, startTime=1733861803673; duration=0sec 2024-12-10T20:16:44,115 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:44,115 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:A 2024-12-10T20:16:44,116 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/b10004148fc6456ea9d3017140b234db is 50, key is test_row_0/C:col10/1733861800101/Put/seqid=0 2024-12-10T20:16:44,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742202_1378 (size=12289) 2024-12-10T20:16:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:44,177 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121039ba1ba33c0149b28e08906b5d65d81a_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121039ba1ba33c0149b28e08906b5d65d81a_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:44,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/145c9931e00c474cb604a7212a20d46b, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:44,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/145c9931e00c474cb604a7212a20d46b is 175, key is test_row_0/A:col10/1733861802365/Put/seqid=0 2024-12-10T20:16:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742203_1379 (size=31105) 2024-12-10T20:16:44,524 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/b10004148fc6456ea9d3017140b234db as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/b10004148fc6456ea9d3017140b234db 2024-12-10T20:16:44,527 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 85784434655d82701a348d7dd2302a0e/C of 85784434655d82701a348d7dd2302a0e into b10004148fc6456ea9d3017140b234db(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:44,527 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:44,527 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/C, priority=12, startTime=1733861803673; duration=0sec 2024-12-10T20:16:44,527 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:44,528 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:C 2024-12-10T20:16:44,584 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/145c9931e00c474cb604a7212a20d46b 2024-12-10T20:16:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/d1cbdc6ffdd9439187fb961749672d64 is 50, key is test_row_0/B:col10/1733861802365/Put/seqid=0 2024-12-10T20:16:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742204_1380 (size=12151) 2024-12-10T20:16:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861864673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861864674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861864675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861864675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861864675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861864787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861864787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861864787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861864787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861864788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861864996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861864997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861864997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861864997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861864997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,005 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/d1cbdc6ffdd9439187fb961749672d64 2024-12-10T20:16:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/88f6e96dbf6e4303955c85670c23d876 is 50, key is test_row_0/C:col10/1733861802365/Put/seqid=0 2024-12-10T20:16:45,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742205_1381 (size=12151) 2024-12-10T20:16:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:45,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861865306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861865306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861865307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861865317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861865317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,421 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/88f6e96dbf6e4303955c85670c23d876 2024-12-10T20:16:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/145c9931e00c474cb604a7212a20d46b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b 2024-12-10T20:16:45,429 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b, entries=150, sequenceid=188, filesize=30.4 K 2024-12-10T20:16:45,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/d1cbdc6ffdd9439187fb961749672d64 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d1cbdc6ffdd9439187fb961749672d64 2024-12-10T20:16:45,433 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d1cbdc6ffdd9439187fb961749672d64, entries=150, sequenceid=188, filesize=11.9 K 2024-12-10T20:16:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/88f6e96dbf6e4303955c85670c23d876 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/88f6e96dbf6e4303955c85670c23d876 2024-12-10T20:16:45,437 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/88f6e96dbf6e4303955c85670c23d876, entries=150, sequenceid=188, filesize=11.9 K 2024-12-10T20:16:45,438 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=60.38 KB/61830 for 85784434655d82701a348d7dd2302a0e in 1685ms, sequenceid=188, compaction requested=false 2024-12-10T20:16:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-10T20:16:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-10T20:16:45,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-10T20:16:45,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4490 sec 2024-12-10T20:16:45,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6040 sec 2024-12-10T20:16:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:45,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:45,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b20fa66641f54f8f81431257c1291182_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:45,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742206_1382 (size=14794) 2024-12-10T20:16:45,831 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:45,834 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b20fa66641f54f8f81431257c1291182_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b20fa66641f54f8f81431257c1291182_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:45,836 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/31f63a385b6b4f05b950a3b49a0ae49a, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:45,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/31f63a385b6b4f05b950a3b49a0ae49a is 175, key is test_row_0/A:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:45,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742207_1383 (size=39749) 2024-12-10T20:16:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861865851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861865851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861865859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861865859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861865860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861865967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861865967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861865977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861865977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861865977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861866177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861866177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861866186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861866186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861866186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,245 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/31f63a385b6b4f05b950a3b49a0ae49a 2024-12-10T20:16:46,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/71aee4bf24894980a165f62f250eb977 is 50, key is test_row_0/B:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:46,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742208_1384 (size=12151) 2024-12-10T20:16:46,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861866487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861866487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861866497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861866506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861866506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:46,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/71aee4bf24894980a165f62f250eb977 2024-12-10T20:16:46,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/119b8c40ca014141825dc31e4393c660 is 50, key is test_row_0/C:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:46,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742209_1385 (size=12151) 2024-12-10T20:16:47,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861867007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:47,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861867007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:47,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861867017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861867026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861867026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:47,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/119b8c40ca014141825dc31e4393c660 2024-12-10T20:16:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/31f63a385b6b4f05b950a3b49a0ae49a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a 2024-12-10T20:16:47,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a, entries=200, sequenceid=204, filesize=38.8 K 2024-12-10T20:16:47,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/71aee4bf24894980a165f62f250eb977 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/71aee4bf24894980a165f62f250eb977 2024-12-10T20:16:47,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/71aee4bf24894980a165f62f250eb977, entries=150, sequenceid=204, filesize=11.9 K 2024-12-10T20:16:47,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/119b8c40ca014141825dc31e4393c660 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/119b8c40ca014141825dc31e4393c660 2024-12-10T20:16:47,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/119b8c40ca014141825dc31e4393c660, entries=150, sequenceid=204, filesize=11.9 K 2024-12-10T20:16:47,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 85784434655d82701a348d7dd2302a0e in 1257ms, sequenceid=204, compaction requested=true 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:47,079 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:47,079 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102097 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/B is initiating minor compaction (all files) 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/A is initiating minor compaction (all files) 2024-12-10T20:16:47,080 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/A in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:47,080 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/B in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:47,080 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/bd9224ec82f54ceaa355c2243a47438f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=99.7 K 2024-12-10T20:16:47,080 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/fa01acc44fab499d8fc5982508f69b60, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d1cbdc6ffdd9439187fb961749672d64, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/71aee4bf24894980a165f62f250eb977] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=35.7 K 2024-12-10T20:16:47,080 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/bd9224ec82f54ceaa355c2243a47438f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a] 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting fa01acc44fab499d8fc5982508f69b60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733861800086 2024-12-10T20:16:47,080 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd9224ec82f54ceaa355c2243a47438f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733861800086 2024-12-10T20:16:47,081 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145c9931e00c474cb604a7212a20d46b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733861802360 2024-12-10T20:16:47,081 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d1cbdc6ffdd9439187fb961749672d64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733861802360 2024-12-10T20:16:47,081 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31f63a385b6b4f05b950a3b49a0ae49a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733861804662 2024-12-10T20:16:47,081 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 71aee4bf24894980a165f62f250eb977, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733861804662 2024-12-10T20:16:47,085 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:47,087 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:47,088 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/12ba6677b1d84778b4bfec5441a932ff is 50, key is test_row_0/B:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:47,093 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210bfa84de3acd94b16951d5cbbfd8e2a1f_85784434655d82701a348d7dd2302a0e store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:47,095 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210bfa84de3acd94b16951d5cbbfd8e2a1f_85784434655d82701a348d7dd2302a0e, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:47,095 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bfa84de3acd94b16951d5cbbfd8e2a1f_85784434655d82701a348d7dd2302a0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:47,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742210_1386 (size=12391) 2024-12-10T20:16:47,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742211_1387 (size=4469) 2024-12-10T20:16:47,100 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#A#compaction#330 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:47,101 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/038ade42891943efa9232fc993f1be94 is 175, key is test_row_0/A:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:47,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742212_1388 (size=31345) 2024-12-10T20:16:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T20:16:47,167 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:47,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-10T20:16:47,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T20:16:47,171 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:47,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T20:16:47,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T20:16:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:47,324 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:16:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102d86c7c600454f9cb3269c71312c8b6a_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861805859/Put/seqid=0 2024-12-10T20:16:47,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742213_1389 (size=12304) 2024-12-10T20:16:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T20:16:47,499 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/12ba6677b1d84778b4bfec5441a932ff as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/12ba6677b1d84778b4bfec5441a932ff 2024-12-10T20:16:47,502 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/B of 85784434655d82701a348d7dd2302a0e into 12ba6677b1d84778b4bfec5441a932ff(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:47,502 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:47,502 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/B, priority=13, startTime=1733861807079; duration=0sec 2024-12-10T20:16:47,502 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:47,502 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:B 2024-12-10T20:16:47,502 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:47,503 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:47,503 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/C is initiating minor compaction (all files) 2024-12-10T20:16:47,503 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/C in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:47,503 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/b10004148fc6456ea9d3017140b234db, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/88f6e96dbf6e4303955c85670c23d876, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/119b8c40ca014141825dc31e4393c660] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=35.7 K 2024-12-10T20:16:47,504 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b10004148fc6456ea9d3017140b234db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733861800086 2024-12-10T20:16:47,504 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f6e96dbf6e4303955c85670c23d876, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733861802360 2024-12-10T20:16:47,505 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 119b8c40ca014141825dc31e4393c660, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733861804662 2024-12-10T20:16:47,507 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/038ade42891943efa9232fc993f1be94 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/038ade42891943efa9232fc993f1be94 2024-12-10T20:16:47,511 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:47,511 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/A of 85784434655d82701a348d7dd2302a0e into 038ade42891943efa9232fc993f1be94(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:47,511 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:47,511 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/A, priority=13, startTime=1733861807079; duration=0sec 2024-12-10T20:16:47,511 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/925a43c37a1845b49ad13d9c4be4ef37 is 50, key is test_row_0/C:col10/1733861804674/Put/seqid=0 2024-12-10T20:16:47,511 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:47,511 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:A 2024-12-10T20:16:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742214_1390 (size=12391) 2024-12-10T20:16:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T20:16:47,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:16:47,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:47,816 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102d86c7c600454f9cb3269c71312c8b6a_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102d86c7c600454f9cb3269c71312c8b6a_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:47,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/74a8ce0ec27848cdb974ddfd370baaa4, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:47,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/74a8ce0ec27848cdb974ddfd370baaa4 is 175, key is test_row_0/A:col10/1733861805859/Put/seqid=0 2024-12-10T20:16:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742215_1391 (size=31105) 2024-12-10T20:16:47,919 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/925a43c37a1845b49ad13d9c4be4ef37 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/925a43c37a1845b49ad13d9c4be4ef37 2024-12-10T20:16:47,922 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/C of 85784434655d82701a348d7dd2302a0e into 925a43c37a1845b49ad13d9c4be4ef37(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:47,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:47,922 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/C, priority=13, startTime=1733861807079; duration=0sec 2024-12-10T20:16:47,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:47,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:C 2024-12-10T20:16:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:48,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861868046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861868046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861868052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861868051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861868054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,054 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:48,054 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861868166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861868167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861868167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,243 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/74a8ce0ec27848cdb974ddfd370baaa4 2024-12-10T20:16:48,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/f14a322868a34f9a8c478a0fbcb57dce is 50, key is test_row_0/B:col10/1733861805859/Put/seqid=0 2024-12-10T20:16:48,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742216_1392 (size=12151) 2024-12-10T20:16:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T20:16:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861868377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861868377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861868377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,652 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/f14a322868a34f9a8c478a0fbcb57dce 2024-12-10T20:16:48,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/0eed99f47cbd424aa7aa8a31737aad3f is 50, key is test_row_0/C:col10/1733861805859/Put/seqid=0 2024-12-10T20:16:48,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742217_1393 (size=12151) 2024-12-10T20:16:48,661 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/0eed99f47cbd424aa7aa8a31737aad3f 2024-12-10T20:16:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/74a8ce0ec27848cdb974ddfd370baaa4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4 2024-12-10T20:16:48,674 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4, entries=150, sequenceid=223, filesize=30.4 K 2024-12-10T20:16:48,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/f14a322868a34f9a8c478a0fbcb57dce as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/f14a322868a34f9a8c478a0fbcb57dce 2024-12-10T20:16:48,680 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/f14a322868a34f9a8c478a0fbcb57dce, entries=150, sequenceid=223, filesize=11.9 K 2024-12-10T20:16:48,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/0eed99f47cbd424aa7aa8a31737aad3f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0eed99f47cbd424aa7aa8a31737aad3f 2024-12-10T20:16:48,686 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0eed99f47cbd424aa7aa8a31737aad3f, entries=150, sequenceid=223, filesize=11.9 K 2024-12-10T20:16:48,686 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 85784434655d82701a348d7dd2302a0e in 1363ms, sequenceid=223, compaction requested=false 2024-12-10T20:16:48,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:48,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:48,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-10T20:16:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-10T20:16:48,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-10T20:16:48,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-12-10T20:16:48,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.5220 sec 2024-12-10T20:16:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:48,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T20:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:48,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f849997eda1e4075a1b40a4475e3f105_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:48,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742218_1394 (size=14794) 2024-12-10T20:16:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861868718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861868720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861868725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861868836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861868836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:48,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861868837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861869047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861869048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861869048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:49,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f849997eda1e4075a1b40a4475e3f105_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f849997eda1e4075a1b40a4475e3f105_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:49,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/88e94dce70c94203b29c06597c162cb3, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:49,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/88e94dce70c94203b29c06597c162cb3 is 175, key is test_row_0/A:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:49,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742219_1395 (size=39749) 2024-12-10T20:16:49,111 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/88e94dce70c94203b29c06597c162cb3 2024-12-10T20:16:49,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ce59327e83e3426faccb1aaacde49227 is 50, key is test_row_0/B:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742220_1396 (size=12151) 2024-12-10T20:16:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T20:16:49,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:49,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-10T20:16:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:49,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:49,319 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:49,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861869356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861869356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861869367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:49,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T20:16:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ce59327e83e3426faccb1aaacde49227 2024-12-10T20:16:49,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/64da9ec746de4bb6bcd67cb51a9a8388 is 50, key is test_row_0/C:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742221_1397 (size=12151) 2024-12-10T20:16:49,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T20:16:49,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:49,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:49,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T20:16:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861869866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861869877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861869877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:49,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T20:16:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:49,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/64da9ec746de4bb6bcd67cb51a9a8388 2024-12-10T20:16:49,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/88e94dce70c94203b29c06597c162cb3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3 2024-12-10T20:16:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:49,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3, entries=200, sequenceid=240, filesize=38.8 K 2024-12-10T20:16:49,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ce59327e83e3426faccb1aaacde49227 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ce59327e83e3426faccb1aaacde49227 2024-12-10T20:16:49,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ce59327e83e3426faccb1aaacde49227, entries=150, sequenceid=240, filesize=11.9 K 2024-12-10T20:16:49,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/64da9ec746de4bb6bcd67cb51a9a8388 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/64da9ec746de4bb6bcd67cb51a9a8388 2024-12-10T20:16:49,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/64da9ec746de4bb6bcd67cb51a9a8388, entries=150, sequenceid=240, filesize=11.9 K 2024-12-10T20:16:49,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 85784434655d82701a348d7dd2302a0e in 1267ms, sequenceid=240, compaction requested=true 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:49,958 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:49,958 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:49,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:49,959 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:49,959 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/A is initiating minor compaction (all files) 2024-12-10T20:16:49,959 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/A in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,959 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/038ade42891943efa9232fc993f1be94, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=99.8 K 2024-12-10T20:16:49,959 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,959 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/038ade42891943efa9232fc993f1be94, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3] 2024-12-10T20:16:49,959 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:49,959 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/B is initiating minor compaction (all files) 2024-12-10T20:16:49,959 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/B in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,959 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/12ba6677b1d84778b4bfec5441a932ff, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/f14a322868a34f9a8c478a0fbcb57dce, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ce59327e83e3426faccb1aaacde49227] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=35.8 K 2024-12-10T20:16:49,960 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 038ade42891943efa9232fc993f1be94, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733861804662 2024-12-10T20:16:49,960 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 12ba6677b1d84778b4bfec5441a932ff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733861804662 2024-12-10T20:16:49,960 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a8ce0ec27848cdb974ddfd370baaa4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733861805841 2024-12-10T20:16:49,960 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f14a322868a34f9a8c478a0fbcb57dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733861805841 2024-12-10T20:16:49,960 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e94dce70c94203b29c06597c162cb3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733861808049 2024-12-10T20:16:49,960 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ce59327e83e3426faccb1aaacde49227, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733861808051 2024-12-10T20:16:49,966 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:49,967 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:49,968 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210cba3319f6b2547e3afdd287496e1bc65_85784434655d82701a348d7dd2302a0e store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:49,968 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/91acdf066bbb4419b5781416680cbd98 is 50, key is test_row_0/B:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:49,969 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210cba3319f6b2547e3afdd287496e1bc65_85784434655d82701a348d7dd2302a0e, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:49,970 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cba3319f6b2547e3afdd287496e1bc65_85784434655d82701a348d7dd2302a0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:49,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742222_1398 (size=12493) 2024-12-10T20:16:49,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742223_1399 (size=4469) 2024-12-10T20:16:49,980 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/91acdf066bbb4419b5781416680cbd98 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/91acdf066bbb4419b5781416680cbd98 2024-12-10T20:16:49,982 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#A#compaction#340 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:49,982 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/448a13a0b7374129943b276b40397a65 is 175, key is test_row_0/A:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:49,986 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/B of 85784434655d82701a348d7dd2302a0e into 91acdf066bbb4419b5781416680cbd98(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:49,986 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:49,986 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/B, priority=13, startTime=1733861809958; duration=0sec 2024-12-10T20:16:49,986 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:49,986 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:B 2024-12-10T20:16:49,986 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742224_1400 (size=31447) 2024-12-10T20:16:49,988 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:49,988 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/C is initiating minor compaction (all files) 2024-12-10T20:16:49,988 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/C in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:49,988 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/925a43c37a1845b49ad13d9c4be4ef37, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0eed99f47cbd424aa7aa8a31737aad3f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/64da9ec746de4bb6bcd67cb51a9a8388] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=35.8 K 2024-12-10T20:16:49,989 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 925a43c37a1845b49ad13d9c4be4ef37, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733861804662 2024-12-10T20:16:49,989 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eed99f47cbd424aa7aa8a31737aad3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733861805841 2024-12-10T20:16:49,990 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 64da9ec746de4bb6bcd67cb51a9a8388, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733861808051 2024-12-10T20:16:49,992 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/448a13a0b7374129943b276b40397a65 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/448a13a0b7374129943b276b40397a65 2024-12-10T20:16:49,998 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/A of 85784434655d82701a348d7dd2302a0e into 448a13a0b7374129943b276b40397a65(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:49,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:49,998 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T20:16:49,998 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/A, priority=13, startTime=1733861809958; duration=0sec 2024-12-10T20:16:49,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:49,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:A 2024-12-10T20:16:49,998 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/08938b46d0b4413fab6f473aab14a31a is 50, key is test_row_0/C:col10/1733861808690/Put/seqid=0 2024-12-10T20:16:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742225_1401 (size=12493) 2024-12-10T20:16:50,008 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/08938b46d0b4413fab6f473aab14a31a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/08938b46d0b4413fab6f473aab14a31a 2024-12-10T20:16:50,014 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/C of 85784434655d82701a348d7dd2302a0e into 08938b46d0b4413fab6f473aab14a31a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:50,014 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:50,014 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/C, priority=13, startTime=1733861809958; duration=0sec 2024-12-10T20:16:50,014 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:50,014 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:C 2024-12-10T20:16:50,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:50,081 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:50,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210201332227952472e9f742d63c3ce200c_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861808711/Put/seqid=0 2024-12-10T20:16:50,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742226_1402 (size=12354) 2024-12-10T20:16:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861870135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861870135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861870247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861870247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:50,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861870457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861870457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:50,503 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210201332227952472e9f742d63c3ce200c_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210201332227952472e9f742d63c3ce200c_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:50,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e64fabdb6f424618b9bab45a19c447c0, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:50,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e64fabdb6f424618b9bab45a19c447c0 is 175, key is test_row_0/A:col10/1733861808711/Put/seqid=0 2024-12-10T20:16:50,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742227_1403 (size=31155) 2024-12-10T20:16:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861870767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861870767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861870876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,882 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861870886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,890 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861870897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:50,902 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:50,907 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e64fabdb6f424618b9bab45a19c447c0 2024-12-10T20:16:50,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/886b2972227544a289a1dda9410f23d8 is 50, key is test_row_0/B:col10/1733861808711/Put/seqid=0 2024-12-10T20:16:50,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742228_1404 (size=12201) 2024-12-10T20:16:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861871276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861871276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:51,317 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/886b2972227544a289a1dda9410f23d8 2024-12-10T20:16:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/bae0e38cc95a475bbca0895ee11d7d39 is 50, key is test_row_0/C:col10/1733861808711/Put/seqid=0 2024-12-10T20:16:51,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742229_1405 (size=12201) 2024-12-10T20:16:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:51,726 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/bae0e38cc95a475bbca0895ee11d7d39 2024-12-10T20:16:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e64fabdb6f424618b9bab45a19c447c0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0 2024-12-10T20:16:51,732 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0, entries=150, sequenceid=261, filesize=30.4 K 2024-12-10T20:16:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/886b2972227544a289a1dda9410f23d8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/886b2972227544a289a1dda9410f23d8 2024-12-10T20:16:51,735 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/886b2972227544a289a1dda9410f23d8, entries=150, sequenceid=261, filesize=11.9 K 2024-12-10T20:16:51,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/bae0e38cc95a475bbca0895ee11d7d39 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bae0e38cc95a475bbca0895ee11d7d39 2024-12-10T20:16:51,740 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bae0e38cc95a475bbca0895ee11d7d39, entries=150, sequenceid=261, filesize=11.9 K 2024-12-10T20:16:51,741 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 85784434655d82701a348d7dd2302a0e in 1660ms, sequenceid=261, compaction requested=false 2024-12-10T20:16:51,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:51,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:51,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-10T20:16:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-10T20:16:51,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-10T20:16:51,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4230 sec 2024-12-10T20:16:51,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.4270 sec 2024-12-10T20:16:51,950 DEBUG [regionserver/03a2ae042c95:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-12-10T20:16:52,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/bd9224ec82f54ceaa355c2243a47438f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/038ade42891943efa9232fc993f1be94, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3] to archive 2024-12-10T20:16:52,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:52,004 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/f8afbaa04d41449aac89252999d19641 2024-12-10T20:16:52,004 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/038ade42891943efa9232fc993f1be94 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/038ade42891943efa9232fc993f1be94 2024-12-10T20:16:52,004 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/31f63a385b6b4f05b950a3b49a0ae49a 2024-12-10T20:16:52,005 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/145c9931e00c474cb604a7212a20d46b 2024-12-10T20:16:52,005 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3320f7c91be7416fbaa82b1510577a68 2024-12-10T20:16:52,005 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/342744ced1ab44519a555e88c58349be 2024-12-10T20:16:52,005 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/8baa94f095254051b6355ddd2365a9e0 2024-12-10T20:16:52,005 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/bd9224ec82f54ceaa355c2243a47438f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/bd9224ec82f54ceaa355c2243a47438f 2024-12-10T20:16:52,006 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/74a8ce0ec27848cdb974ddfd370baaa4 2024-12-10T20:16:52,007 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/88e94dce70c94203b29c06597c162cb3 2024-12-10T20:16:52,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/06595f36c7ac4bbfa4c3d4540b73db47, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/5ec623328bc1498dad1b02c5b59ea3de, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/a82fa46223fa4dc2bd1e186215f14678, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/fa01acc44fab499d8fc5982508f69b60, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d6ca1f6f812d4143871bd553e59b3027, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d1cbdc6ffdd9439187fb961749672d64, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/12ba6677b1d84778b4bfec5441a932ff, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/71aee4bf24894980a165f62f250eb977, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/f14a322868a34f9a8c478a0fbcb57dce, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ce59327e83e3426faccb1aaacde49227] to archive 2024-12-10T20:16:52,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/5ec623328bc1498dad1b02c5b59ea3de to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/5ec623328bc1498dad1b02c5b59ea3de 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/a82fa46223fa4dc2bd1e186215f14678 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/a82fa46223fa4dc2bd1e186215f14678 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/06595f36c7ac4bbfa4c3d4540b73db47 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/06595f36c7ac4bbfa4c3d4540b73db47 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d6ca1f6f812d4143871bd553e59b3027 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d6ca1f6f812d4143871bd553e59b3027 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/fa01acc44fab499d8fc5982508f69b60 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/fa01acc44fab499d8fc5982508f69b60 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d1cbdc6ffdd9439187fb961749672d64 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/d1cbdc6ffdd9439187fb961749672d64 2024-12-10T20:16:52,013 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/71aee4bf24894980a165f62f250eb977 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/71aee4bf24894980a165f62f250eb977 2024-12-10T20:16:52,014 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/12ba6677b1d84778b4bfec5441a932ff to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/12ba6677b1d84778b4bfec5441a932ff 2024-12-10T20:16:52,014 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ce59327e83e3426faccb1aaacde49227 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ce59327e83e3426faccb1aaacde49227 2024-12-10T20:16:52,014 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/f14a322868a34f9a8c478a0fbcb57dce to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/f14a322868a34f9a8c478a0fbcb57dce 2024-12-10T20:16:52,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0e23ea73d39d44d59a93079222df9c09, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/66afedf28b4b474bb6cda898090ee433, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d5d7a168bd354b6badafeab29e36b5f5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/b10004148fc6456ea9d3017140b234db, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b91da2447fe4e60859108387ada97df, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/88f6e96dbf6e4303955c85670c23d876, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/925a43c37a1845b49ad13d9c4be4ef37, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/119b8c40ca014141825dc31e4393c660, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0eed99f47cbd424aa7aa8a31737aad3f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/64da9ec746de4bb6bcd67cb51a9a8388] to archive 2024-12-10T20:16:52,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:16:52,018 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d5d7a168bd354b6badafeab29e36b5f5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d5d7a168bd354b6badafeab29e36b5f5 2024-12-10T20:16:52,018 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b91da2447fe4e60859108387ada97df to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b91da2447fe4e60859108387ada97df 2024-12-10T20:16:52,019 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0e23ea73d39d44d59a93079222df9c09 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0e23ea73d39d44d59a93079222df9c09 2024-12-10T20:16:52,019 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/b10004148fc6456ea9d3017140b234db to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/b10004148fc6456ea9d3017140b234db 2024-12-10T20:16:52,019 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/66afedf28b4b474bb6cda898090ee433 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/66afedf28b4b474bb6cda898090ee433 2024-12-10T20:16:52,019 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/88f6e96dbf6e4303955c85670c23d876 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/88f6e96dbf6e4303955c85670c23d876 2024-12-10T20:16:52,019 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/925a43c37a1845b49ad13d9c4be4ef37 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/925a43c37a1845b49ad13d9c4be4ef37 2024-12-10T20:16:52,019 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/119b8c40ca014141825dc31e4393c660 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/119b8c40ca014141825dc31e4393c660 2024-12-10T20:16:52,020 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0eed99f47cbd424aa7aa8a31737aad3f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/0eed99f47cbd424aa7aa8a31737aad3f 2024-12-10T20:16:52,020 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/64da9ec746de4bb6bcd67cb51a9a8388 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/64da9ec746de4bb6bcd67cb51a9a8388 2024-12-10T20:16:52,024 WARN [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-10T20:16:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:52,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:52,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:52,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dfebb593af30471b86d2789079a55fc7_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:52,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742230_1406 (size=17534) 2024-12-10T20:16:52,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:52,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dfebb593af30471b86d2789079a55fc7_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dfebb593af30471b86d2789079a55fc7_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:52,318 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/05fabc14203e47f68f65db2bcb8777fb, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:52,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/05fabc14203e47f68f65db2bcb8777fb is 175, key is test_row_0/A:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:52,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742231_1407 (size=48639) 2024-12-10T20:16:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861872347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861872355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861872467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861872467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861872677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861872677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/05fabc14203e47f68f65db2bcb8777fb 2024-12-10T20:16:52,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ec0cd70ade2b46489e45a0b9da0929da is 50, key is test_row_0/B:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:52,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742232_1408 (size=12301) 2024-12-10T20:16:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861872897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861872897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,902 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,902 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861872926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,933 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861872987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861872987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:53,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ec0cd70ade2b46489e45a0b9da0929da 2024-12-10T20:16:53,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/bdca933a8a3f4b1aac8eee2fa066b0d4 is 50, key is test_row_0/C:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:53,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742233_1409 (size=12301) 2024-12-10T20:16:53,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T20:16:53,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:53,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-10T20:16:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T20:16:53,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:53,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:53,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861873508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861873508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:53,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/bdca933a8a3f4b1aac8eee2fa066b0d4 2024-12-10T20:16:53,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T20:16:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:53,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T20:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T20:16:53,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/05fabc14203e47f68f65db2bcb8777fb as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb 2024-12-10T20:16:53,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb, entries=250, sequenceid=277, filesize=47.5 K 2024-12-10T20:16:53,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ec0cd70ade2b46489e45a0b9da0929da as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ec0cd70ade2b46489e45a0b9da0929da 2024-12-10T20:16:53,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ec0cd70ade2b46489e45a0b9da0929da, entries=150, sequenceid=277, filesize=12.0 K 2024-12-10T20:16:53,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/bdca933a8a3f4b1aac8eee2fa066b0d4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bdca933a8a3f4b1aac8eee2fa066b0d4 2024-12-10T20:16:53,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bdca933a8a3f4b1aac8eee2fa066b0d4, entries=150, sequenceid=277, filesize=12.0 K 2024-12-10T20:16:53,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 85784434655d82701a348d7dd2302a0e in 1618ms, sequenceid=277, compaction requested=true 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:53,920 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:53,920 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:53,921 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:53,921 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:53,921 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/A is initiating minor compaction (all files) 2024-12-10T20:16:53,921 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/B is initiating minor compaction (all files) 2024-12-10T20:16:53,921 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/B in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,921 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/A in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,921 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/91acdf066bbb4419b5781416680cbd98, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/886b2972227544a289a1dda9410f23d8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ec0cd70ade2b46489e45a0b9da0929da] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=36.1 K 2024-12-10T20:16:53,921 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/448a13a0b7374129943b276b40397a65, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=108.6 K 2024-12-10T20:16:53,921 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,921 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/448a13a0b7374129943b276b40397a65, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb] 2024-12-10T20:16:53,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 91acdf066bbb4419b5781416680cbd98, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733861808051 2024-12-10T20:16:53,922 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448a13a0b7374129943b276b40397a65, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733861808051 2024-12-10T20:16:53,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 886b2972227544a289a1dda9410f23d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733861808711 2024-12-10T20:16:53,922 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e64fabdb6f424618b9bab45a19c447c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733861808711 2024-12-10T20:16:53,922 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0cd70ade2b46489e45a0b9da0929da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733861810134 2024-12-10T20:16:53,922 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05fabc14203e47f68f65db2bcb8777fb, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733861810110 2024-12-10T20:16:53,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T20:16:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:53,936 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T20:16:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:53,940 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:53,941 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ecd7dee9790f4078b64950f9cc331854 is 50, key is test_row_0/B:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:53,945 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f7a689747b7b4b17a7eff93283c268b1_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861812346/Put/seqid=0 2024-12-10T20:16:53,958 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210d8dc951accd647759914cc9278cbb3a4_85784434655d82701a348d7dd2302a0e store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:53,959 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210d8dc951accd647759914cc9278cbb3a4_85784434655d82701a348d7dd2302a0e, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:53,960 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d8dc951accd647759914cc9278cbb3a4_85784434655d82701a348d7dd2302a0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:53,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742234_1410 (size=12404) 2024-12-10T20:16:53,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742235_1411 (size=12454) 2024-12-10T20:16:53,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:53,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742236_1412 (size=4469) 2024-12-10T20:16:53,989 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f7a689747b7b4b17a7eff93283c268b1_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f7a689747b7b4b17a7eff93283c268b1_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:53,990 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#A#compaction#349 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/405838eb06af4c79beb13b55d4f5315f, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:53,991 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/1a750e3898b34e6a8c55b8c04c92ee41 is 175, key is test_row_0/A:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:53,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/405838eb06af4c79beb13b55d4f5315f is 175, key is test_row_0/A:col10/1733861812346/Put/seqid=0 2024-12-10T20:16:54,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742238_1414 (size=31255) 2024-12-10T20:16:54,002 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/405838eb06af4c79beb13b55d4f5315f 2024-12-10T20:16:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/b0301b3f0598462bb80fc8aa432bf89d is 50, key is test_row_0/B:col10/1733861812346/Put/seqid=0 2024-12-10T20:16:54,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742237_1413 (size=31358) 2024-12-10T20:16:54,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742239_1415 (size=12301) 2024-12-10T20:16:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T20:16:54,384 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/ecd7dee9790f4078b64950f9cc331854 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ecd7dee9790f4078b64950f9cc331854 2024-12-10T20:16:54,387 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/B of 85784434655d82701a348d7dd2302a0e into ecd7dee9790f4078b64950f9cc331854(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:54,387 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:54,387 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/B, priority=13, startTime=1733861813920; duration=0sec 2024-12-10T20:16:54,387 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:54,387 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:B 2024-12-10T20:16:54,387 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:54,388 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:54,388 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/C is initiating minor compaction (all files) 2024-12-10T20:16:54,388 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/C in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:54,388 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/08938b46d0b4413fab6f473aab14a31a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bae0e38cc95a475bbca0895ee11d7d39, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bdca933a8a3f4b1aac8eee2fa066b0d4] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=36.1 K 2024-12-10T20:16:54,389 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 08938b46d0b4413fab6f473aab14a31a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733861808051 2024-12-10T20:16:54,389 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting bae0e38cc95a475bbca0895ee11d7d39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733861808711 2024-12-10T20:16:54,389 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting bdca933a8a3f4b1aac8eee2fa066b0d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733861810134 2024-12-10T20:16:54,394 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#C#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:54,395 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/982e5f4c5db14c5b83bc30d9b126885a is 50, key is test_row_0/C:col10/1733861812301/Put/seqid=0 2024-12-10T20:16:54,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742240_1416 (size=12404) 2024-12-10T20:16:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T20:16:54,422 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/1a750e3898b34e6a8c55b8c04c92ee41 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1a750e3898b34e6a8c55b8c04c92ee41 2024-12-10T20:16:54,426 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/A of 85784434655d82701a348d7dd2302a0e into 1a750e3898b34e6a8c55b8c04c92ee41(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:54,426 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:54,426 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/A, priority=13, startTime=1733861813920; duration=0sec 2024-12-10T20:16:54,426 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:54,426 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:A 2024-12-10T20:16:54,427 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/b0301b3f0598462bb80fc8aa432bf89d 2024-12-10T20:16:54,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/623d1a6a9f104112a8d7f9ae0c3593a1 is 50, key is test_row_0/C:col10/1733861812346/Put/seqid=0 2024-12-10T20:16:54,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742241_1417 (size=12301) 2024-12-10T20:16:54,437 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/623d1a6a9f104112a8d7f9ae0c3593a1 2024-12-10T20:16:54,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/405838eb06af4c79beb13b55d4f5315f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f 2024-12-10T20:16:54,445 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f, entries=150, sequenceid=293, filesize=30.5 K 2024-12-10T20:16:54,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/b0301b3f0598462bb80fc8aa432bf89d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/b0301b3f0598462bb80fc8aa432bf89d 2024-12-10T20:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,449 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/b0301b3f0598462bb80fc8aa432bf89d, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T20:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/623d1a6a9f104112a8d7f9ae0c3593a1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/623d1a6a9f104112a8d7f9ae0c3593a1 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/623d1a6a9f104112a8d7f9ae0c3593a1, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 85784434655d82701a348d7dd2302a0e in 517ms, sequenceid=293, compaction requested=false 2024-12-10T20:16:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-10T20:16:54,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 975 msec 2024-12-10T20:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 979 msec 2024-12-10T20:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:54,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:16:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:54,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100bc38890425549079f67e49692cf9cb2_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:54,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742242_1418 (size=17534) 2024-12-10T20:16:54,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861814694, max timestamp=1733861814730Num uniques 859; 2024-12-10T20:16:54,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:54,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861814694, max timestamp=1733861814730Num uniques 859; 2024-12-10T20:16:54,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:54,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861814694, max timestamp=1733861814730Num uniques 859; 2024-12-10T20:16:54,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:16:54,802 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/982e5f4c5db14c5b83bc30d9b126885a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/982e5f4c5db14c5b83bc30d9b126885a 2024-12-10T20:16:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861874797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:54,812 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/C of 85784434655d82701a348d7dd2302a0e into 982e5f4c5db14c5b83bc30d9b126885a(size=12.1 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:54,813 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:54,813 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/C, priority=13, startTime=1733861813920; duration=0sec 2024-12-10T20:16:54,813 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:54,813 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:C 2024-12-10T20:16:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861874808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:54,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861874916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T20:16:54,928 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:54,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:16:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-10T20:16:54,930 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:16:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:54,931 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:16:54,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:16:54,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861874927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:55,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:55,067 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100bc38890425549079f67e49692cf9cb2_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100bc38890425549079f67e49692cf9cb2_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:55,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/408b8d21e32d402395f5451cac4cda80, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:55,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/408b8d21e32d402395f5451cac4cda80 is 175, key is test_row_0/A:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:55,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742243_1419 (size=48639) 2024-12-10T20:16:55,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861875137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861875147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:55,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861875446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861875457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,480 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/408b8d21e32d402395f5451cac4cda80 2024-12-10T20:16:55,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/e35e016a02624554b6674f787e583de6 is 50, key is test_row_0/B:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:55,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742244_1420 (size=12301) 2024-12-10T20:16:55,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:55,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/e35e016a02624554b6674f787e583de6 2024-12-10T20:16:55,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/29fe759cd8cc4ccc89605c194fdfda3c is 50, key is test_row_0/C:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:55,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742245_1421 (size=12301) 2024-12-10T20:16:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861875966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861875976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:55,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:55,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:55,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:56,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:16:56,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/29fe759cd8cc4ccc89605c194fdfda3c 2024-12-10T20:16:56,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/408b8d21e32d402395f5451cac4cda80 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80 2024-12-10T20:16:56,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80, entries=250, sequenceid=305, filesize=47.5 K 2024-12-10T20:16:56,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/e35e016a02624554b6674f787e583de6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/e35e016a02624554b6674f787e583de6 2024-12-10T20:16:56,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/e35e016a02624554b6674f787e583de6, entries=150, sequenceid=305, filesize=12.0 K 2024-12-10T20:16:56,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/29fe759cd8cc4ccc89605c194fdfda3c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/29fe759cd8cc4ccc89605c194fdfda3c 2024-12-10T20:16:56,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/29fe759cd8cc4ccc89605c194fdfda3c, entries=150, sequenceid=305, filesize=12.0 K 2024-12-10T20:16:56,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 85784434655d82701a348d7dd2302a0e in 1667ms, sequenceid=305, compaction requested=true 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:56,323 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:56,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:56,323 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111252 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37006 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/B is initiating minor compaction (all files) 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/A is initiating minor compaction (all files) 2024-12-10T20:16:56,324 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/B in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,324 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/A in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,324 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ecd7dee9790f4078b64950f9cc331854, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/b0301b3f0598462bb80fc8aa432bf89d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/e35e016a02624554b6674f787e583de6] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=36.1 K 2024-12-10T20:16:56,324 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1a750e3898b34e6a8c55b8c04c92ee41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=108.6 K 2024-12-10T20:16:56,324 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1a750e3898b34e6a8c55b8c04c92ee41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80] 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd7dee9790f4078b64950f9cc331854, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733861810134 2024-12-10T20:16:56,324 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a750e3898b34e6a8c55b8c04c92ee41, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733861810134 2024-12-10T20:16:56,325 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b0301b3f0598462bb80fc8aa432bf89d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733861812330 2024-12-10T20:16:56,325 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405838eb06af4c79beb13b55d4f5315f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733861812330 2024-12-10T20:16:56,325 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e35e016a02624554b6674f787e583de6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733861814645 2024-12-10T20:16:56,325 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408b8d21e32d402395f5451cac4cda80, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733861814581 2024-12-10T20:16:56,332 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:56,334 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:56,334 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412104c0dd8f01d674a398c9a0e5e5e83a71a_85784434655d82701a348d7dd2302a0e store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:56,334 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/4e1eb882f5204415b9ebd2c6a895b6b9 is 50, key is test_row_0/B:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:56,336 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412104c0dd8f01d674a398c9a0e5e5e83a71a_85784434655d82701a348d7dd2302a0e, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:56,336 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104c0dd8f01d674a398c9a0e5e5e83a71a_85784434655d82701a348d7dd2302a0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:56,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742246_1422 (size=12507) 2024-12-10T20:16:56,342 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/4e1eb882f5204415b9ebd2c6a895b6b9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/4e1eb882f5204415b9ebd2c6a895b6b9 2024-12-10T20:16:56,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742247_1423 (size=4469) 2024-12-10T20:16:56,346 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#A#compaction#358 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:56,346 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/7656357dfa5b495fa8e3e063aeff9452 is 175, key is test_row_0/A:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:56,346 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/B of 85784434655d82701a348d7dd2302a0e into 4e1eb882f5204415b9ebd2c6a895b6b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:56,346 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:56,346 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/B, priority=13, startTime=1733861816323; duration=0sec 2024-12-10T20:16:56,346 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:56,346 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:B 2024-12-10T20:16:56,346 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:56,348 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37006 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:56,348 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/C is initiating minor compaction (all files) 2024-12-10T20:16:56,348 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/C in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,348 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/982e5f4c5db14c5b83bc30d9b126885a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/623d1a6a9f104112a8d7f9ae0c3593a1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/29fe759cd8cc4ccc89605c194fdfda3c] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=36.1 K 2024-12-10T20:16:56,348 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 982e5f4c5db14c5b83bc30d9b126885a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733861810134 2024-12-10T20:16:56,348 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 623d1a6a9f104112a8d7f9ae0c3593a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733861812330 2024-12-10T20:16:56,349 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 29fe759cd8cc4ccc89605c194fdfda3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733861814645 2024-12-10T20:16:56,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742248_1424 (size=31461) 2024-12-10T20:16:56,357 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/7656357dfa5b495fa8e3e063aeff9452 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/7656357dfa5b495fa8e3e063aeff9452 2024-12-10T20:16:56,358 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:56,359 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/d2bea47c185b4991b7d2cdf823df273b is 50, key is test_row_0/C:col10/1733861814655/Put/seqid=0 2024-12-10T20:16:56,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742249_1425 (size=12507) 2024-12-10T20:16:56,363 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/A of 85784434655d82701a348d7dd2302a0e into 7656357dfa5b495fa8e3e063aeff9452(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:56,363 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:56,363 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/A, priority=13, startTime=1733861816323; duration=0sec 2024-12-10T20:16:56,363 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:56,363 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:A 2024-12-10T20:16:56,366 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/d2bea47c185b4991b7d2cdf823df273b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d2bea47c185b4991b7d2cdf823df273b 2024-12-10T20:16:56,371 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/C of 85784434655d82701a348d7dd2302a0e into d2bea47c185b4991b7d2cdf823df273b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:16:56,371 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:56,371 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/C, priority=13, startTime=1733861816323; duration=0sec 2024-12-10T20:16:56,371 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:56,371 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:C 2024-12-10T20:16:56,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T20:16:56,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:56,456 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T20:16:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:56,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:16:56,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210747279edf0674a4f906aaa2e2d52135a_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861814761/Put/seqid=0 2024-12-10T20:16:56,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742250_1426 (size=12454) 2024-12-10T20:16:56,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:56,869 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210747279edf0674a4f906aaa2e2d52135a_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210747279edf0674a4f906aaa2e2d52135a_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:56,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/ced1fc4c61ad494fa2b89bb7a4f03ce5, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:56,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/ced1fc4c61ad494fa2b89bb7a4f03ce5 is 175, key is test_row_0/A:col10/1733861814761/Put/seqid=0 2024-12-10T20:16:56,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742251_1427 (size=31255) 2024-12-10T20:16:56,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. as already flushing 2024-12-10T20:16:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861876968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:56,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861876969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53360 deadline: 1733861876977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:56,984 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:56,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861876987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:56,998 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861876997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,003 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:57,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861877087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861877087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,273 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/ced1fc4c61ad494fa2b89bb7a4f03ce5 2024-12-10T20:16:57,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/72d7f21ea844493d9f948f21ec840745 is 50, key is test_row_0/B:col10/1733861814761/Put/seqid=0 2024-12-10T20:16:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742252_1428 (size=12301) 2024-12-10T20:16:57,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861877297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861877297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861877607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861877607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:57,683 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/72d7f21ea844493d9f948f21ec840745 2024-12-10T20:16:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/cbfd3f604a9f4d29b3231c62115fbddc is 50, key is test_row_0/C:col10/1733861814761/Put/seqid=0 2024-12-10T20:16:57,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742253_1429 (size=12301) 2024-12-10T20:16:58,092 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/cbfd3f604a9f4d29b3231c62115fbddc 2024-12-10T20:16:58,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/ced1fc4c61ad494fa2b89bb7a4f03ce5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5 2024-12-10T20:16:58,099 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5, entries=150, sequenceid=345, filesize=30.5 K 2024-12-10T20:16:58,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/72d7f21ea844493d9f948f21ec840745 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/72d7f21ea844493d9f948f21ec840745 2024-12-10T20:16:58,103 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/72d7f21ea844493d9f948f21ec840745, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T20:16:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/cbfd3f604a9f4d29b3231c62115fbddc as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/cbfd3f604a9f4d29b3231c62115fbddc 2024-12-10T20:16:58,106 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/cbfd3f604a9f4d29b3231c62115fbddc, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T20:16:58,107 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 85784434655d82701a348d7dd2302a0e in 1651ms, sequenceid=345, compaction requested=false 2024-12-10T20:16:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-10T20:16:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-10T20:16:58,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-10T20:16:58,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1770 sec 2024-12-10T20:16:58,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.1820 sec 2024-12-10T20:16:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:58,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:16:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:16:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:58,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:16:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:16:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:16:58,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a0fdaadbe60e4b18ad8762cef4041164_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861818130/Put/seqid=0 2024-12-10T20:16:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742254_1430 (size=17534) 2024-12-10T20:16:58,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861878203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861878205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861878317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861878317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861878527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:16:58,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861878537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,545 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a0fdaadbe60e4b18ad8762cef4041164_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0fdaadbe60e4b18ad8762cef4041164_85784434655d82701a348d7dd2302a0e 2024-12-10T20:16:58,546 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e38505ffa2c140a28b90f35b6230d55c, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:58,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e38505ffa2c140a28b90f35b6230d55c is 175, key is test_row_0/A:col10/1733861818130/Put/seqid=0 2024-12-10T20:16:58,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742255_1431 (size=48639) 2024-12-10T20:16:58,686 INFO [Thread-1641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1641. 2024-12-10T20:16:58,686 DEBUG [Thread-1641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:58,686 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,686 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:58,688 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-12-10T20:16:58,688 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:58,688 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,689 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:58,690 INFO [Thread-1637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1637. 2024-12-10T20:16:58,690 DEBUG [Thread-1637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:58,690 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,690 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:58,693 INFO [Thread-1643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1643. 2024-12-10T20:16:58,693 DEBUG [Thread-1643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:58,693 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,693 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:58,695 INFO [Thread-1639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1639. 2024-12-10T20:16:58,695 DEBUG [Thread-1639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:16:58,695 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,695 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:16:58,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:16:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861878838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861878847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:58,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e38505ffa2c140a28b90f35b6230d55c 2024-12-10T20:16:58,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/2333d4687f664b93a87303a5d520d582 is 50, key is test_row_0/B:col10/1733861818130/Put/seqid=0 2024-12-10T20:16:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742256_1432 (size=12301) 2024-12-10T20:16:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53414 deadline: 1733861879008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:59,009 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53376 deadline: 1733861879017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:59,019 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 85784434655d82701a348d7dd2302a0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:16:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T20:16:59,098 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:16:59,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53402 deadline: 1733861879358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:59,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/2333d4687f664b93a87303a5d520d582 2024-12-10T20:16:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53416 deadline: 1733861879456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:16:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:16:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:16:59,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/8b9e3a14e5a54491a27f8cfe3d65713b is 50, key is test_row_0/C:col10/1733861818130/Put/seqid=0 2024-12-10T20:16:59,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742257_1433 (size=12301) 2024-12-10T20:16:59,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/8b9e3a14e5a54491a27f8cfe3d65713b 2024-12-10T20:16:59,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/e38505ffa2c140a28b90f35b6230d55c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c 2024-12-10T20:16:59,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c, entries=250, sequenceid=357, filesize=47.5 K 2024-12-10T20:16:59,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/2333d4687f664b93a87303a5d520d582 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/2333d4687f664b93a87303a5d520d582 2024-12-10T20:16:59,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/2333d4687f664b93a87303a5d520d582, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T20:16:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/8b9e3a14e5a54491a27f8cfe3d65713b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b9e3a14e5a54491a27f8cfe3d65713b 2024-12-10T20:16:59,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b9e3a14e5a54491a27f8cfe3d65713b, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T20:16:59,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 85784434655d82701a348d7dd2302a0e in 1759ms, sequenceid=357, compaction requested=true 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:59,890 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:16:59,890 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 85784434655d82701a348d7dd2302a0e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:16:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/A is initiating minor compaction (all files) 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/B is initiating minor compaction (all files) 2024-12-10T20:16:59,891 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/A in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:59,891 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/B in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:59,891 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/7656357dfa5b495fa8e3e063aeff9452, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=108.7 K 2024-12-10T20:16:59,891 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/4e1eb882f5204415b9ebd2c6a895b6b9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/72d7f21ea844493d9f948f21ec840745, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/2333d4687f664b93a87303a5d520d582] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=36.2 K 2024-12-10T20:16:59,891 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/7656357dfa5b495fa8e3e063aeff9452, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c] 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1eb882f5204415b9ebd2c6a895b6b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733861814645 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7656357dfa5b495fa8e3e063aeff9452, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733861814645 2024-12-10T20:16:59,891 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d7f21ea844493d9f948f21ec840745, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733861814761 2024-12-10T20:16:59,892 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ced1fc4c61ad494fa2b89bb7a4f03ce5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733861814761 2024-12-10T20:16:59,892 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 2333d4687f664b93a87303a5d520d582, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733861816952 2024-12-10T20:16:59,892 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38505ffa2c140a28b90f35b6230d55c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733861816952 2024-12-10T20:16:59,896 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:59,898 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412101bec024ceb294d1683a99b082e16b1b4_85784434655d82701a348d7dd2302a0e store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:59,899 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:16:59,899 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/170c75d6bab24e6290345144b7d85942 is 50, key is test_row_0/B:col10/1733861818130/Put/seqid=0 2024-12-10T20:16:59,902 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412101bec024ceb294d1683a99b082e16b1b4_85784434655d82701a348d7dd2302a0e, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:59,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742258_1434 (size=12609) 2024-12-10T20:16:59,902 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101bec024ceb294d1683a99b082e16b1b4_85784434655d82701a348d7dd2302a0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:16:59,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742259_1435 (size=4469) 2024-12-10T20:17:00,305 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:00,306 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/3f9d71eb74e040dfaa171a57e640a4f9 is 175, key is test_row_0/A:col10/1733861818130/Put/seqid=0 2024-12-10T20:17:00,308 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/170c75d6bab24e6290345144b7d85942 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/170c75d6bab24e6290345144b7d85942 2024-12-10T20:17:00,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742260_1436 (size=31563) 2024-12-10T20:17:00,313 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/B of 85784434655d82701a348d7dd2302a0e into 170c75d6bab24e6290345144b7d85942(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:00,313 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:17:00,313 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/B, priority=13, startTime=1733861819890; duration=0sec 2024-12-10T20:17:00,313 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:00,313 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:B 2024-12-10T20:17:00,313 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:00,315 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:00,315 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 85784434655d82701a348d7dd2302a0e/C is initiating minor compaction (all files) 2024-12-10T20:17:00,315 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 85784434655d82701a348d7dd2302a0e/C in TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:17:00,315 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d2bea47c185b4991b7d2cdf823df273b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/cbfd3f604a9f4d29b3231c62115fbddc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b9e3a14e5a54491a27f8cfe3d65713b] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp, totalSize=36.2 K 2024-12-10T20:17:00,315 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d2bea47c185b4991b7d2cdf823df273b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733861814645 2024-12-10T20:17:00,316 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting cbfd3f604a9f4d29b3231c62115fbddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733861814761 2024-12-10T20:17:00,316 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b9e3a14e5a54491a27f8cfe3d65713b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733861816952 2024-12-10T20:17:00,322 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 85784434655d82701a348d7dd2302a0e#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:00,322 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/89eb7d7310e3400db778ed83e83ef43a is 50, key is test_row_0/C:col10/1733861818130/Put/seqid=0 2024-12-10T20:17:00,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742261_1437 (size=12609) 2024-12-10T20:17:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:00,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T20:17:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:17:00,370 INFO [Thread-1626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1626. 2024-12-10T20:17:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:00,370 DEBUG [Thread-1626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:17:00,370 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:00,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:00,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:17:00,371 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:00,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:00,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:00,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121072fe2b288440477297832f7c9e3b91bb_85784434655d82701a348d7dd2302a0e is 50, key is test_row_0/A:col10/1733861818176/Put/seqid=0 2024-12-10T20:17:00,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742262_1438 (size=12454) 2024-12-10T20:17:00,470 INFO [Thread-1630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1630. 2024-12-10T20:17:00,470 DEBUG [Thread-1630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:00,470 DEBUG [Thread-1630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:00,471 DEBUG [Thread-1630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:00,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:00,717 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/3f9d71eb74e040dfaa171a57e640a4f9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3f9d71eb74e040dfaa171a57e640a4f9 2024-12-10T20:17:00,724 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/A of 85784434655d82701a348d7dd2302a0e into 3f9d71eb74e040dfaa171a57e640a4f9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:00,724 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:17:00,724 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/A, priority=13, startTime=1733861819890; duration=0sec 2024-12-10T20:17:00,724 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:00,724 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:A 2024-12-10T20:17:00,729 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/89eb7d7310e3400db778ed83e83ef43a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/89eb7d7310e3400db778ed83e83ef43a 2024-12-10T20:17:00,733 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 85784434655d82701a348d7dd2302a0e/C of 85784434655d82701a348d7dd2302a0e into 89eb7d7310e3400db778ed83e83ef43a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:00,733 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:17:00,733 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., storeName=85784434655d82701a348d7dd2302a0e/C, priority=13, startTime=1733861819890; duration=0sec 2024-12-10T20:17:00,734 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:00,734 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 85784434655d82701a348d7dd2302a0e:C 2024-12-10T20:17:00,782 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:00,787 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121072fe2b288440477297832f7c9e3b91bb_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072fe2b288440477297832f7c9e3b91bb_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:00,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/2733022c2ef046229b1805f58f700b75, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:17:00,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/2733022c2ef046229b1805f58f700b75 is 175, key is test_row_0/A:col10/1733861818176/Put/seqid=0 2024-12-10T20:17:00,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742263_1439 (size=31255) 2024-12-10T20:17:01,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/2733022c2ef046229b1805f58f700b75 2024-12-10T20:17:01,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/33d9c23a7fe2494e81b0b627d01cbf30 is 50, key is test_row_0/B:col10/1733861818176/Put/seqid=0 2024-12-10T20:17:01,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742264_1440 (size=12301) 2024-12-10T20:17:01,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/33d9c23a7fe2494e81b0b627d01cbf30 2024-12-10T20:17:01,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/126a41cde5be455a86fa6b49c000a393 is 50, key is test_row_0/C:col10/1733861818176/Put/seqid=0 2024-12-10T20:17:01,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742265_1441 (size=12301) 2024-12-10T20:17:02,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/126a41cde5be455a86fa6b49c000a393 2024-12-10T20:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/2733022c2ef046229b1805f58f700b75 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/2733022c2ef046229b1805f58f700b75 2024-12-10T20:17:02,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/2733022c2ef046229b1805f58f700b75, entries=150, sequenceid=379, filesize=30.5 K 2024-12-10T20:17:02,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/33d9c23a7fe2494e81b0b627d01cbf30 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/33d9c23a7fe2494e81b0b627d01cbf30 2024-12-10T20:17:02,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/33d9c23a7fe2494e81b0b627d01cbf30, entries=150, sequenceid=379, filesize=12.0 K 2024-12-10T20:17:02,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/126a41cde5be455a86fa6b49c000a393 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/126a41cde5be455a86fa6b49c000a393 2024-12-10T20:17:02,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/126a41cde5be455a86fa6b49c000a393, entries=150, sequenceid=379, filesize=12.0 K 2024-12-10T20:17:02,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=6.71 KB/6870 for 85784434655d82701a348d7dd2302a0e in 1681ms, sequenceid=379, compaction requested=false 2024-12-10T20:17:02,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 85784434655d82701a348d7dd2302a0e: 2024-12-10T20:17:03,051 INFO [Thread-1632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1632. 2024-12-10T20:17:03,051 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-12-10T20:17:03,051 DEBUG [Thread-1632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:03,051 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:03,051 DEBUG [Thread-1632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:03,051 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:03,051 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:03,051 DEBUG [Thread-1632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:03,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:03,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:07,009 INFO [Thread-1628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1628. 2024-12-10T20:17:07,009 DEBUG [Thread-1628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:07,009 DEBUG [Thread-1628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:07,009 DEBUG [Thread-1628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:07,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4152 rows 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1372 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4116 rows 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1370 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4110 rows 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1385 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4155 rows 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1376 2024-12-10T20:17:07,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4128 rows 2024-12-10T20:17:07,010 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:17:07,010 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:17:07,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:07,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:07,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:07,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T20:17:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T20:17:07,016 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861827016"}]},"ts":"1733861827016"} 2024-12-10T20:17:07,018 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T20:17:07,018 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T20:17:07,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:17:07,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, UNASSIGN}] 2024-12-10T20:17:07,021 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, UNASSIGN 2024-12-10T20:17:07,021 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:07,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, UNASSIGN because future has completed 2024-12-10T20:17:07,023 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:17:07,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:17:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T20:17:07,176 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:07,176 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:17:07,176 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 85784434655d82701a348d7dd2302a0e, disabling compactions & flushes 2024-12-10T20:17:07,177 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:17:07,177 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:17:07,177 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. after waiting 0 ms 2024-12-10T20:17:07,177 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:17:07,177 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 85784434655d82701a348d7dd2302a0e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T20:17:07,178 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=A 2024-12-10T20:17:07,178 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:07,178 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=B 2024-12-10T20:17:07,178 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:07,178 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 85784434655d82701a348d7dd2302a0e, store=C 2024-12-10T20:17:07,178 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:07,185 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210474ee1c859964c13ac205b6ca44168ab_85784434655d82701a348d7dd2302a0e is 50, key is test_row_1/A:col10/1733861823048/Put/seqid=0 2024-12-10T20:17:07,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742266_1442 (size=9914) 2024-12-10T20:17:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T20:17:07,590 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:07,595 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210474ee1c859964c13ac205b6ca44168ab_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210474ee1c859964c13ac205b6ca44168ab_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:07,596 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/1389d3fc8d3e43fbb04af47f2eeb975d, store: [table=TestAcidGuarantees family=A region=85784434655d82701a348d7dd2302a0e] 2024-12-10T20:17:07,597 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/1389d3fc8d3e43fbb04af47f2eeb975d is 175, key is test_row_1/A:col10/1733861823048/Put/seqid=0 2024-12-10T20:17:07,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742267_1443 (size=22561) 2024-12-10T20:17:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T20:17:08,002 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/1389d3fc8d3e43fbb04af47f2eeb975d 2024-12-10T20:17:08,013 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/17315d3592974feda2768b9c7eaa181d is 50, key is test_row_1/B:col10/1733861823048/Put/seqid=0 2024-12-10T20:17:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742268_1444 (size=9857) 2024-12-10T20:17:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T20:17:08,418 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/17315d3592974feda2768b9c7eaa181d 2024-12-10T20:17:08,428 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/a3173e3030c04ddda9fe97bef6c5436b is 50, key is test_row_1/C:col10/1733861823048/Put/seqid=0 2024-12-10T20:17:08,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742269_1445 (size=9857) 2024-12-10T20:17:08,832 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/a3173e3030c04ddda9fe97bef6c5436b 2024-12-10T20:17:08,837 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/A/1389d3fc8d3e43fbb04af47f2eeb975d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1389d3fc8d3e43fbb04af47f2eeb975d 2024-12-10T20:17:08,842 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1389d3fc8d3e43fbb04af47f2eeb975d, entries=100, sequenceid=388, filesize=22.0 K 2024-12-10T20:17:08,843 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/B/17315d3592974feda2768b9c7eaa181d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/17315d3592974feda2768b9c7eaa181d 2024-12-10T20:17:08,848 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/17315d3592974feda2768b9c7eaa181d, entries=100, sequenceid=388, filesize=9.6 K 2024-12-10T20:17:08,849 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/.tmp/C/a3173e3030c04ddda9fe97bef6c5436b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/a3173e3030c04ddda9fe97bef6c5436b 2024-12-10T20:17:08,852 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/a3173e3030c04ddda9fe97bef6c5436b, entries=100, sequenceid=388, filesize=9.6 K 2024-12-10T20:17:08,852 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 85784434655d82701a348d7dd2302a0e in 1675ms, sequenceid=388, compaction requested=true 2024-12-10T20:17:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/448a13a0b7374129943b276b40397a65, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1a750e3898b34e6a8c55b8c04c92ee41, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/7656357dfa5b495fa8e3e063aeff9452, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c] to archive 2024-12-10T20:17:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/448a13a0b7374129943b276b40397a65 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/448a13a0b7374129943b276b40397a65 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1a750e3898b34e6a8c55b8c04c92ee41 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1a750e3898b34e6a8c55b8c04c92ee41 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/7656357dfa5b495fa8e3e063aeff9452 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/7656357dfa5b495fa8e3e063aeff9452 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/405838eb06af4c79beb13b55d4f5315f 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/ced1fc4c61ad494fa2b89bb7a4f03ce5 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e64fabdb6f424618b9bab45a19c447c0 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/408b8d21e32d402395f5451cac4cda80 2024-12-10T20:17:08,855 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/05fabc14203e47f68f65db2bcb8777fb 2024-12-10T20:17:08,856 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/e38505ffa2c140a28b90f35b6230d55c 2024-12-10T20:17:08,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/91acdf066bbb4419b5781416680cbd98, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/886b2972227544a289a1dda9410f23d8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ecd7dee9790f4078b64950f9cc331854, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ec0cd70ade2b46489e45a0b9da0929da, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/b0301b3f0598462bb80fc8aa432bf89d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/4e1eb882f5204415b9ebd2c6a895b6b9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/e35e016a02624554b6674f787e583de6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/72d7f21ea844493d9f948f21ec840745, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/2333d4687f664b93a87303a5d520d582] to archive 2024-12-10T20:17:08,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:17:08,859 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/886b2972227544a289a1dda9410f23d8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/886b2972227544a289a1dda9410f23d8 2024-12-10T20:17:08,859 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/91acdf066bbb4419b5781416680cbd98 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/91acdf066bbb4419b5781416680cbd98 2024-12-10T20:17:08,859 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/4e1eb882f5204415b9ebd2c6a895b6b9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/4e1eb882f5204415b9ebd2c6a895b6b9 2024-12-10T20:17:08,859 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ecd7dee9790f4078b64950f9cc331854 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ecd7dee9790f4078b64950f9cc331854 2024-12-10T20:17:08,859 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ec0cd70ade2b46489e45a0b9da0929da to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/ec0cd70ade2b46489e45a0b9da0929da 2024-12-10T20:17:08,860 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/e35e016a02624554b6674f787e583de6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/e35e016a02624554b6674f787e583de6 2024-12-10T20:17:08,860 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/b0301b3f0598462bb80fc8aa432bf89d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/b0301b3f0598462bb80fc8aa432bf89d 2024-12-10T20:17:08,860 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/72d7f21ea844493d9f948f21ec840745 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/72d7f21ea844493d9f948f21ec840745 2024-12-10T20:17:08,860 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/2333d4687f664b93a87303a5d520d582 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/2333d4687f664b93a87303a5d520d582 2024-12-10T20:17:08,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/08938b46d0b4413fab6f473aab14a31a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bae0e38cc95a475bbca0895ee11d7d39, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/982e5f4c5db14c5b83bc30d9b126885a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bdca933a8a3f4b1aac8eee2fa066b0d4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/623d1a6a9f104112a8d7f9ae0c3593a1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d2bea47c185b4991b7d2cdf823df273b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/29fe759cd8cc4ccc89605c194fdfda3c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/cbfd3f604a9f4d29b3231c62115fbddc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b9e3a14e5a54491a27f8cfe3d65713b] to archive 2024-12-10T20:17:08,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:17:08,864 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/08938b46d0b4413fab6f473aab14a31a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/08938b46d0b4413fab6f473aab14a31a 2024-12-10T20:17:08,864 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/623d1a6a9f104112a8d7f9ae0c3593a1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/623d1a6a9f104112a8d7f9ae0c3593a1 2024-12-10T20:17:08,864 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/982e5f4c5db14c5b83bc30d9b126885a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/982e5f4c5db14c5b83bc30d9b126885a 2024-12-10T20:17:08,864 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bae0e38cc95a475bbca0895ee11d7d39 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bae0e38cc95a475bbca0895ee11d7d39 2024-12-10T20:17:08,864 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/29fe759cd8cc4ccc89605c194fdfda3c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/29fe759cd8cc4ccc89605c194fdfda3c 2024-12-10T20:17:08,864 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bdca933a8a3f4b1aac8eee2fa066b0d4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/bdca933a8a3f4b1aac8eee2fa066b0d4 2024-12-10T20:17:08,865 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d2bea47c185b4991b7d2cdf823df273b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/d2bea47c185b4991b7d2cdf823df273b 2024-12-10T20:17:08,865 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/cbfd3f604a9f4d29b3231c62115fbddc to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/cbfd3f604a9f4d29b3231c62115fbddc 2024-12-10T20:17:08,865 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b9e3a14e5a54491a27f8cfe3d65713b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/8b9e3a14e5a54491a27f8cfe3d65713b 2024-12-10T20:17:08,868 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/recovered.edits/391.seqid, newMaxSeqId=391, maxSeqId=4 2024-12-10T20:17:08,869 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. 2024-12-10T20:17:08,869 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 85784434655d82701a348d7dd2302a0e: Waiting for close lock at 1733861827176Running coprocessor pre-close hooks at 1733861827176Disabling compacts and flushes for region at 1733861827176Disabling writes for close at 1733861827177 (+1 ms)Obtaining lock to block concurrent updates at 1733861827177Preparing flush snapshotting stores in 85784434655d82701a348d7dd2302a0e at 1733861827177Finished memstore snapshotting TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733861827178 (+1 ms)Flushing stores of TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e. at 1733861827179 (+1 ms)Flushing 85784434655d82701a348d7dd2302a0e/A: creating writer at 1733861827179Flushing mob file 85784434655d82701a348d7dd2302a0e/A: appending metadata at 1733861827185 (+6 ms)Flushing mob file 85784434655d82701a348d7dd2302a0e/A: closing flushed file at 1733861827185Flushing 85784434655d82701a348d7dd2302a0e/A: appending metadata at 1733861827596 (+411 ms)Flushing 85784434655d82701a348d7dd2302a0e/A: closing flushed file at 1733861827596Flushing 85784434655d82701a348d7dd2302a0e/B: creating writer at 1733861828008 (+412 ms)Flushing 85784434655d82701a348d7dd2302a0e/B: appending metadata at 1733861828012 (+4 ms)Flushing 85784434655d82701a348d7dd2302a0e/B: closing flushed file at 1733861828013 (+1 ms)Flushing 85784434655d82701a348d7dd2302a0e/C: creating writer at 1733861828424 (+411 ms)Flushing 85784434655d82701a348d7dd2302a0e/C: appending metadata at 1733861828427 (+3 ms)Flushing 85784434655d82701a348d7dd2302a0e/C: closing flushed file at 1733861828427Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@655afa71: reopening flushed file at 1733861828836 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eb67686: reopening flushed file at 1733861828842 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@789df45c: reopening flushed file at 1733861828848 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 85784434655d82701a348d7dd2302a0e in 1675ms, sequenceid=388, compaction requested=true at 1733861828852 (+4 ms)Writing region close event to WAL at 1733861828866 (+14 ms)Running coprocessor post-close hooks at 1733861828869 (+3 ms)Closed at 1733861828869 2024-12-10T20:17:08,871 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:08,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=85784434655d82701a348d7dd2302a0e, regionState=CLOSED 2024-12-10T20:17:08,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:17:08,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-10T20:17:08,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 85784434655d82701a348d7dd2302a0e, server=03a2ae042c95,38133,1733861689839 in 1.8500 sec 2024-12-10T20:17:08,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-12-10T20:17:08,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=85784434655d82701a348d7dd2302a0e, UNASSIGN in 1.8540 sec 2024-12-10T20:17:08,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-10T20:17:08,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-10T20:17:08,878 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861828878"}]},"ts":"1733861828878"} 2024-12-10T20:17:08,879 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T20:17:08,879 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T20:17:08,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8670 sec 2024-12-10T20:17:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T20:17:09,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:09,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T20:17:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,169 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T20:17:09,170 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,172 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,174 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/recovered.edits] 2024-12-10T20:17:09,177 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/2733022c2ef046229b1805f58f700b75 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/2733022c2ef046229b1805f58f700b75 2024-12-10T20:17:09,177 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1389d3fc8d3e43fbb04af47f2eeb975d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/1389d3fc8d3e43fbb04af47f2eeb975d 2024-12-10T20:17:09,177 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3f9d71eb74e040dfaa171a57e640a4f9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/A/3f9d71eb74e040dfaa171a57e640a4f9 2024-12-10T20:17:09,179 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/170c75d6bab24e6290345144b7d85942 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/170c75d6bab24e6290345144b7d85942 2024-12-10T20:17:09,179 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/33d9c23a7fe2494e81b0b627d01cbf30 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/33d9c23a7fe2494e81b0b627d01cbf30 2024-12-10T20:17:09,179 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/17315d3592974feda2768b9c7eaa181d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/B/17315d3592974feda2768b9c7eaa181d 2024-12-10T20:17:09,181 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/89eb7d7310e3400db778ed83e83ef43a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/89eb7d7310e3400db778ed83e83ef43a 2024-12-10T20:17:09,181 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/126a41cde5be455a86fa6b49c000a393 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/126a41cde5be455a86fa6b49c000a393 2024-12-10T20:17:09,181 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/a3173e3030c04ddda9fe97bef6c5436b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/C/a3173e3030c04ddda9fe97bef6c5436b 2024-12-10T20:17:09,183 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/recovered.edits/391.seqid to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e/recovered.edits/391.seqid 2024-12-10T20:17:09,183 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,183 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T20:17:09,184 DEBUG [PEWorker-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T20:17:09,184 DEBUG [PEWorker-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T20:17:09,188 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100bc38890425549079f67e49692cf9cb2_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100bc38890425549079f67e49692cf9cb2_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100e29f195d20841d9bdd64eba470a80b2_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100e29f195d20841d9bdd64eba470a80b2_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121039ba1ba33c0149b28e08906b5d65d81a_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121039ba1ba33c0149b28e08906b5d65d81a_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210201332227952472e9f742d63c3ce200c_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210201332227952472e9f742d63c3ce200c_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121032ad348415994f09bcf8c06e1f4e9690_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121032ad348415994f09bcf8c06e1f4e9690_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102d86c7c600454f9cb3269c71312c8b6a_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102d86c7c600454f9cb3269c71312c8b6a_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103ed76eabfe15456e95853e0889c4b19d_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103ed76eabfe15456e95853e0889c4b19d_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,189 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210474ee1c859964c13ac205b6ca44168ab_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210474ee1c859964c13ac205b6ca44168ab_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072fe2b288440477297832f7c9e3b91bb_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072fe2b288440477297832f7c9e3b91bb_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210747279edf0674a4f906aaa2e2d52135a_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210747279edf0674a4f906aaa2e2d52135a_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0fdaadbe60e4b18ad8762cef4041164_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0fdaadbe60e4b18ad8762cef4041164_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b20fa66641f54f8f81431257c1291182_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b20fa66641f54f8f81431257c1291182_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dfebb593af30471b86d2789079a55fc7_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dfebb593af30471b86d2789079a55fc7_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f7a689747b7b4b17a7eff93283c268b1_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f7a689747b7b4b17a7eff93283c268b1_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f849997eda1e4075a1b40a4475e3f105_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f849997eda1e4075a1b40a4475e3f105_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fd92be41865141abbfaa7688f97f9a8b_85784434655d82701a348d7dd2302a0e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fd92be41865141abbfaa7688f97f9a8b_85784434655d82701a348d7dd2302a0e 2024-12-10T20:17:09,190 DEBUG [PEWorker-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T20:17:09,192 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,193 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T20:17:09,195 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T20:17:09,196 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,196 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T20:17:09,196 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733861829196"}]},"ts":"9223372036854775807"} 2024-12-10T20:17:09,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T20:17:09,198 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 85784434655d82701a348d7dd2302a0e, NAME => 'TestAcidGuarantees,,1733861795440.85784434655d82701a348d7dd2302a0e.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T20:17:09,198 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T20:17:09,198 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733861829198"}]},"ts":"9223372036854775807"} 2024-12-10T20:17:09,200 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T20:17:09,200 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-10T20:17:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T20:17:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T20:17:09,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:09,293 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=253 (was 251) - Thread LEAK? -, OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=839 (was 798) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3123 (was 1337) - AvailableMemoryMB LEAK? - 2024-12-10T20:17:09,303 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=253, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=839, ProcessCount=11, AvailableMemoryMB=3123 2024-12-10T20:17:09,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:17:09,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:17:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:09,306 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:17:09,307 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:09,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-12-10T20:17:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-10T20:17:09,307 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:17:09,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742270_1446 (size=1061) 2024-12-10T20:17:09,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T20:17:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-10T20:17:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-10T20:17:09,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:17:09,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742271_1447 (size=53) 2024-12-10T20:17:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-10T20:17:10,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:17:10,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3412e855b2a7dd08270299896873abf4, disabling compactions & flushes 2024-12-10T20:17:10,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. after waiting 0 ms 2024-12-10T20:17:10,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,121 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3412e855b2a7dd08270299896873abf4: Waiting for close lock at 1733861830121Disabling compacts and flushes for region at 1733861830121Disabling writes for close at 1733861830121Writing region close event to WAL at 1733861830121Closed at 1733861830121 2024-12-10T20:17:10,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:17:10,125 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733861830124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733861830124"}]},"ts":"1733861830124"} 2024-12-10T20:17:10,128 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:17:10,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:17:10,131 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861830130"}]},"ts":"1733861830130"} 2024-12-10T20:17:10,133 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T20:17:10,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, ASSIGN}] 2024-12-10T20:17:10,135 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, ASSIGN 2024-12-10T20:17:10,136 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, ASSIGN; state=OFFLINE, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=false 2024-12-10T20:17:10,286 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=3412e855b2a7dd08270299896873abf4, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:10,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, ASSIGN because future has completed 2024-12-10T20:17:10,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:17:10,443 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,443 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:17:10,445 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,445 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:17:10,445 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,445 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,446 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-10T20:17:10,448 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:10,448 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3412e855b2a7dd08270299896873abf4 columnFamilyName A 2024-12-10T20:17:10,448 DEBUG [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:10,449 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.HStore(327): Store=3412e855b2a7dd08270299896873abf4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:10,449 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,451 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:10,451 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3412e855b2a7dd08270299896873abf4 columnFamilyName B 2024-12-10T20:17:10,451 DEBUG [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:10,451 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.HStore(327): Store=3412e855b2a7dd08270299896873abf4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:10,451 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,453 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:10,453 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3412e855b2a7dd08270299896873abf4 columnFamilyName C 2024-12-10T20:17:10,453 DEBUG [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:10,454 INFO [StoreOpener-3412e855b2a7dd08270299896873abf4-1 {}] regionserver.HStore(327): Store=3412e855b2a7dd08270299896873abf4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:10,454 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,454 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,455 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,455 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,456 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,456 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,457 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:17:10,459 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,461 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:17:10,461 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 3412e855b2a7dd08270299896873abf4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67652049, jitterRate=0.00809408724308014}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:17:10,461 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:10,462 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 3412e855b2a7dd08270299896873abf4: Running coprocessor pre-open hook at 1733861830445Writing region info on filesystem at 1733861830445Initializing all the Stores at 1733861830446 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861830446Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861830446Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861830446Cleaning up temporary data from old regions at 1733861830456 (+10 ms)Running coprocessor post-open hooks at 1733861830461 (+5 ms)Region opened successfully at 1733861830462 (+1 ms) 2024-12-10T20:17:10,463 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., pid=108, masterSystemTime=1733861830439 2024-12-10T20:17:10,464 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,464 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:10,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=3412e855b2a7dd08270299896873abf4, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:10,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:17:10,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-10T20:17:10,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 in 178 msec 2024-12-10T20:17:10,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-10T20:17:10,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, ASSIGN in 335 msec 2024-12-10T20:17:10,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:17:10,470 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861830470"}]},"ts":"1733861830470"} 2024-12-10T20:17:10,471 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T20:17:10,472 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:17:10,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-12-10T20:17:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-10T20:17:11,457 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:11,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,458 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd9ef12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,460 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf4086e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,460 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,463 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:17:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:17:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6926be0a 2024-12-10T20:17:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:17:11,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:17:11,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1f545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,467 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e27624e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,468 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9db40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c19380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,471 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e1b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,472 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ced71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b23f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,474 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47598b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,475 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfe3c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7d791a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,476 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,477 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52adfb8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,478 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212e97c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b3a94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,480 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ba282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e67f42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b33a176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,483 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e5fcbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,485 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d2447b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,487 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dc8da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f9fad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304234f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,490 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283a75dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,491 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110fba1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1847540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,494 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@452d0652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,495 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@137a973a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4826bbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:11,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:11,497 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5365df9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:11,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:11,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:11,498 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c80d540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:11,500 DEBUG [Thread-2015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,500 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,500 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,500 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,500 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,509 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,509 DEBUG [Thread-2023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,511 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:11,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:17:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,513 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,513 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:11,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,514 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-10T20:17:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:11,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T20:17:11,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:11,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:11,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:11,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2] 2024-12-10T20:17:11,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/e5b01d7385354631b9c38507e086c8d2 is 50, key is test_row_0/A:col10/1733861831511/Put/seqid=0 2024-12-10T20:17:11,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742272_1448 (size=12001) 2024-12-10T20:17:11,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/e5b01d7385354631b9c38507e086c8d2 2024-12-10T20:17:11,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733861831536, max timestamp=1733861831554Num uniques 859; 2024-12-10T20:17:11,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733861831536, max timestamp=1733861831554Num uniques 859; 2024-12-10T20:17:11,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:17:11,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:17:11,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733861831536, max timestamp=1733861831554Num uniques 859; 2024-12-10T20:17:11,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:17:11,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/33079cfccbb04c43be18165709872258 is 50, key is test_row_0/B:col10/1733861831511/Put/seqid=0 2024-12-10T20:17:11,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861891586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861891587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861891588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742273_1449 (size=12001) 2024-12-10T20:17:11,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861891598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861891606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T20:17:11,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T20:17:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:11,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861891721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861891721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861891722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861891722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861891727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T20:17:11,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T20:17:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:11,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:11,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861891938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861891938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861891938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861891938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861891941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T20:17:12,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:12,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:12,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:12,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:12,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:12,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/33079cfccbb04c43be18165709872258 2024-12-10T20:17:12,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/7c688f388eed4fe5998acf1587e7d643 is 50, key is test_row_0/C:col10/1733861831511/Put/seqid=0 2024-12-10T20:17:12,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742274_1450 (size=12001) 2024-12-10T20:17:12,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/7c688f388eed4fe5998acf1587e7d643 2024-12-10T20:17:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/e5b01d7385354631b9c38507e086c8d2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e5b01d7385354631b9c38507e086c8d2 2024-12-10T20:17:12,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e5b01d7385354631b9c38507e086c8d2, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T20:17:12,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/33079cfccbb04c43be18165709872258 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/33079cfccbb04c43be18165709872258 2024-12-10T20:17:12,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/33079cfccbb04c43be18165709872258, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T20:17:12,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/7c688f388eed4fe5998acf1587e7d643 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/7c688f388eed4fe5998acf1587e7d643 2024-12-10T20:17:12,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/7c688f388eed4fe5998acf1587e7d643, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T20:17:12,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3412e855b2a7dd08270299896873abf4 in 587ms, sequenceid=12, compaction requested=false 2024-12-10T20:17:12,099 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T20:17:12,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T20:17:12,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T20:17:12,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:12,160 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-10T20:17:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/de1ddbe5927e44ef9b85c82aa258981d is 50, key is test_row_0/A:col10/1733861831584/Put/seqid=0 2024-12-10T20:17:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742275_1451 (size=12001) 2024-12-10T20:17:12,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:12,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861892280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861892291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861892293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861892298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861892298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861892407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861892416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861892417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861892417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861892422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,579 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/de1ddbe5927e44ef9b85c82aa258981d 2024-12-10T20:17:12,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/6c6e403cd23f4c9aadf267290e15588b is 50, key is test_row_0/B:col10/1733861831584/Put/seqid=0 2024-12-10T20:17:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861892619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742276_1452 (size=12001) 2024-12-10T20:17:12,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861892637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861892637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861892637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861892637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T20:17:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861892937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861892947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861892947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861892947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861892956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,034 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/6c6e403cd23f4c9aadf267290e15588b 2024-12-10T20:17:13,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/1340701cfb01485d95f0a43b2da7e6ab is 50, key is test_row_0/C:col10/1733861831584/Put/seqid=0 2024-12-10T20:17:13,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742277_1453 (size=12001) 2024-12-10T20:17:13,443 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/1340701cfb01485d95f0a43b2da7e6ab 2024-12-10T20:17:13,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/de1ddbe5927e44ef9b85c82aa258981d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/de1ddbe5927e44ef9b85c82aa258981d 2024-12-10T20:17:13,452 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/de1ddbe5927e44ef9b85c82aa258981d, entries=150, sequenceid=46, filesize=11.7 K 2024-12-10T20:17:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/6c6e403cd23f4c9aadf267290e15588b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6c6e403cd23f4c9aadf267290e15588b 2024-12-10T20:17:13,455 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6c6e403cd23f4c9aadf267290e15588b, entries=150, sequenceid=46, filesize=11.7 K 2024-12-10T20:17:13,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/1340701cfb01485d95f0a43b2da7e6ab as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/1340701cfb01485d95f0a43b2da7e6ab 2024-12-10T20:17:13,458 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/1340701cfb01485d95f0a43b2da7e6ab, entries=150, sequenceid=46, filesize=11.7 K 2024-12-10T20:17:13,459 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 3412e855b2a7dd08270299896873abf4 in 1299ms, sequenceid=46, compaction requested=false 2024-12-10T20:17:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-10T20:17:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-10T20:17:13,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-10T20:17:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:13,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-12-10T20:17:13,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:17:13,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:13,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:13,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:13,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:13,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:13,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:13,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9500 sec 2024-12-10T20:17:13,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f51a7fbe126a49718927ede816fc7abc is 50, key is test_row_0/A:col10/1733861833461/Put/seqid=0 2024-12-10T20:17:13,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742278_1454 (size=14341) 2024-12-10T20:17:13,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861893488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861893494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861893495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861893495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861893496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861893607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861893607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861893607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861893607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861893607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T20:17:13,677 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:13,678 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-10T20:17:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T20:17:13,679 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:13,680 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:13,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T20:17:13,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-10T20:17:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:13,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:13,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861893826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861893827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861893827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861893827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861893827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:13,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f51a7fbe126a49718927ede816fc7abc 2024-12-10T20:17:13,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/1b8649c192fb47d7873b571d28fb34f3 is 50, key is test_row_0/B:col10/1733861833461/Put/seqid=0 2024-12-10T20:17:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742279_1455 (size=12001) 2024-12-10T20:17:13,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/1b8649c192fb47d7873b571d28fb34f3 2024-12-10T20:17:13,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/e75efeea180d46f6a954503eef1da939 is 50, key is test_row_0/C:col10/1733861833461/Put/seqid=0 2024-12-10T20:17:13,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-10T20:17:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:13,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T20:17:14,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742280_1456 (size=12001) 2024-12-10T20:17:14,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/e75efeea180d46f6a954503eef1da939 2024-12-10T20:17:14,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f51a7fbe126a49718927ede816fc7abc as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f51a7fbe126a49718927ede816fc7abc 2024-12-10T20:17:14,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f51a7fbe126a49718927ede816fc7abc, entries=200, sequenceid=59, filesize=14.0 K 2024-12-10T20:17:14,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/1b8649c192fb47d7873b571d28fb34f3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/1b8649c192fb47d7873b571d28fb34f3 2024-12-10T20:17:14,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/1b8649c192fb47d7873b571d28fb34f3, entries=150, sequenceid=59, filesize=11.7 K 2024-12-10T20:17:14,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/e75efeea180d46f6a954503eef1da939 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/e75efeea180d46f6a954503eef1da939 2024-12-10T20:17:14,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/e75efeea180d46f6a954503eef1da939, entries=150, sequenceid=59, filesize=11.7 K 2024-12-10T20:17:14,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3412e855b2a7dd08270299896873abf4 in 573ms, sequenceid=59, compaction requested=true 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:14,035 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:14,035 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:14,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:14,036 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:14,036 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:14,036 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:14,036 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:14,036 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:14,036 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:14,036 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e5b01d7385354631b9c38507e086c8d2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/de1ddbe5927e44ef9b85c82aa258981d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f51a7fbe126a49718927ede816fc7abc] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=37.4 K 2024-12-10T20:17:14,036 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/33079cfccbb04c43be18165709872258, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6c6e403cd23f4c9aadf267290e15588b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/1b8649c192fb47d7873b571d28fb34f3] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=35.2 K 2024-12-10T20:17:14,037 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 33079cfccbb04c43be18165709872258, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733861831507 2024-12-10T20:17:14,037 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b01d7385354631b9c38507e086c8d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733861831507 2024-12-10T20:17:14,037 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6e403cd23f4c9aadf267290e15588b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733861831584 2024-12-10T20:17:14,037 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting de1ddbe5927e44ef9b85c82aa258981d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733861831584 2024-12-10T20:17:14,037 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f51a7fbe126a49718927ede816fc7abc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733861832267 2024-12-10T20:17:14,037 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8649c192fb47d7873b571d28fb34f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733861832267 2024-12-10T20:17:14,055 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:14,055 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/a468d96a903c4d65895a7ee90f544f79 is 50, key is test_row_0/A:col10/1733861833461/Put/seqid=0 2024-12-10T20:17:14,066 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:14,066 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/6414f99180bc4e36a245b6d9febacd2d is 50, key is test_row_0/B:col10/1733861833461/Put/seqid=0 2024-12-10T20:17:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742281_1457 (size=12104) 2024-12-10T20:17:14,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742282_1458 (size=12104) 2024-12-10T20:17:14,081 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/6414f99180bc4e36a245b6d9febacd2d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6414f99180bc4e36a245b6d9febacd2d 2024-12-10T20:17:14,087 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 6414f99180bc4e36a245b6d9febacd2d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:14,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:14,087 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861834035; duration=0sec 2024-12-10T20:17:14,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:14,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:14,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:14,088 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:14,088 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:14,088 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:14,088 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/7c688f388eed4fe5998acf1587e7d643, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/1340701cfb01485d95f0a43b2da7e6ab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/e75efeea180d46f6a954503eef1da939] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=35.2 K 2024-12-10T20:17:14,089 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c688f388eed4fe5998acf1587e7d643, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733861831507 2024-12-10T20:17:14,089 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 1340701cfb01485d95f0a43b2da7e6ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733861831584 2024-12-10T20:17:14,089 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e75efeea180d46f6a954503eef1da939, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733861832267 2024-12-10T20:17:14,102 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:14,102 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/a7be9eedfdd649a782b0374d13c6d13d is 50, key is test_row_0/C:col10/1733861833461/Put/seqid=0 2024-12-10T20:17:14,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742283_1459 (size=12104) 2024-12-10T20:17:14,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:14,138 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/7bce20a35f874e57aede8090c41f24d1 is 50, key is test_row_0/A:col10/1733861833483/Put/seqid=0 2024-12-10T20:17:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:14,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:14,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742284_1460 (size=12001) 2024-12-10T20:17:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861894162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861894168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861894169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861894170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861894171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861894277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861894287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861894287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861894287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861894288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T20:17:14,390 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:17:14,474 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/a468d96a903c4d65895a7ee90f544f79 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a468d96a903c4d65895a7ee90f544f79 2024-12-10T20:17:14,477 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into a468d96a903c4d65895a7ee90f544f79(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:14,478 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:14,478 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861834035; duration=0sec 2024-12-10T20:17:14,478 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:14,478 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861894496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861894497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861894497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861894497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861894497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,518 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/a7be9eedfdd649a782b0374d13c6d13d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/a7be9eedfdd649a782b0374d13c6d13d 2024-12-10T20:17:14,521 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into a7be9eedfdd649a782b0374d13c6d13d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:14,521 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:14,521 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861834035; duration=0sec 2024-12-10T20:17:14,521 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:14,521 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:14,557 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/7bce20a35f874e57aede8090c41f24d1 2024-12-10T20:17:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/d36fcd58f76a45a480b2c395fdf012db is 50, key is test_row_0/B:col10/1733861833483/Put/seqid=0 2024-12-10T20:17:14,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742285_1461 (size=12001) 2024-12-10T20:17:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861894806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861894806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861894806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861894807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861894807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T20:17:14,983 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/d36fcd58f76a45a480b2c395fdf012db 2024-12-10T20:17:14,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8b019d1d4bf74f53825065bbdf87549f is 50, key is test_row_0/C:col10/1733861833483/Put/seqid=0 2024-12-10T20:17:14,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742286_1462 (size=12001) 2024-12-10T20:17:14,994 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8b019d1d4bf74f53825065bbdf87549f 2024-12-10T20:17:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/7bce20a35f874e57aede8090c41f24d1 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/7bce20a35f874e57aede8090c41f24d1 2024-12-10T20:17:14,999 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/7bce20a35f874e57aede8090c41f24d1, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T20:17:15,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/d36fcd58f76a45a480b2c395fdf012db as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d36fcd58f76a45a480b2c395fdf012db 2024-12-10T20:17:15,003 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d36fcd58f76a45a480b2c395fdf012db, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T20:17:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8b019d1d4bf74f53825065bbdf87549f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8b019d1d4bf74f53825065bbdf87549f 2024-12-10T20:17:15,008 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8b019d1d4bf74f53825065bbdf87549f, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T20:17:15,009 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3412e855b2a7dd08270299896873abf4 in 870ms, sequenceid=79, compaction requested=false 2024-12-10T20:17:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-10T20:17:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-10T20:17:15,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-10T20:17:15,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3300 sec 2024-12-10T20:17:15,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3340 sec 2024-12-10T20:17:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:15,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T20:17:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:15,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/fdf68235157b4a0e90bc8ddce9d56aac is 50, key is test_row_0/A:col10/1733861834162/Put/seqid=0 2024-12-10T20:17:15,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742287_1463 (size=14341) 2024-12-10T20:17:15,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/fdf68235157b4a0e90bc8ddce9d56aac 2024-12-10T20:17:15,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/e19b779735f549988bbe386617c74762 is 50, key is test_row_0/B:col10/1733861834162/Put/seqid=0 2024-12-10T20:17:15,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742288_1464 (size=12001) 2024-12-10T20:17:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861895344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861895345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861895350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861895351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861895353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861895457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861895457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861895458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861895467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861895467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861895667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861895667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861895667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861895677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861895677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:15,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/e19b779735f549988bbe386617c74762 2024-12-10T20:17:15,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/5d0451da506d41cca03702092acef5f9 is 50, key is test_row_0/C:col10/1733861834162/Put/seqid=0 2024-12-10T20:17:15,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742289_1465 (size=12001) 2024-12-10T20:17:15,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/5d0451da506d41cca03702092acef5f9 2024-12-10T20:17:15,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/fdf68235157b4a0e90bc8ddce9d56aac as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fdf68235157b4a0e90bc8ddce9d56aac 2024-12-10T20:17:15,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fdf68235157b4a0e90bc8ddce9d56aac, entries=200, sequenceid=95, filesize=14.0 K 2024-12-10T20:17:15,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/e19b779735f549988bbe386617c74762 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/e19b779735f549988bbe386617c74762 2024-12-10T20:17:15,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/e19b779735f549988bbe386617c74762, entries=150, sequenceid=95, filesize=11.7 K 2024-12-10T20:17:15,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/5d0451da506d41cca03702092acef5f9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5d0451da506d41cca03702092acef5f9 2024-12-10T20:17:15,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5d0451da506d41cca03702092acef5f9, entries=150, sequenceid=95, filesize=11.7 K 2024-12-10T20:17:15,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3412e855b2a7dd08270299896873abf4 in 454ms, sequenceid=95, compaction requested=true 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:15,775 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:15,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T20:17:15,776 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:15,776 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:15,776 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:15,776 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a468d96a903c4d65895a7ee90f544f79, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/7bce20a35f874e57aede8090c41f24d1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fdf68235157b4a0e90bc8ddce9d56aac] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=37.5 K 2024-12-10T20:17:15,776 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:15,777 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a468d96a903c4d65895a7ee90f544f79, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733861832267 2024-12-10T20:17:15,777 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bce20a35f874e57aede8090c41f24d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733861833483 2024-12-10T20:17:15,777 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf68235157b4a0e90bc8ddce9d56aac, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733861834162 2024-12-10T20:17:15,783 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:15,783 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:15,783 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:15,783 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6414f99180bc4e36a245b6d9febacd2d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d36fcd58f76a45a480b2c395fdf012db, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/e19b779735f549988bbe386617c74762] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=35.3 K 2024-12-10T20:17:15,783 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 6414f99180bc4e36a245b6d9febacd2d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733861832267 2024-12-10T20:17:15,784 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting d36fcd58f76a45a480b2c395fdf012db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733861833483 2024-12-10T20:17:15,784 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting e19b779735f549988bbe386617c74762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733861834162 2024-12-10T20:17:15,791 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:15,791 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:15,791 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/1c72909e98d14372932812b13029fec8 is 50, key is test_row_0/A:col10/1733861834162/Put/seqid=0 2024-12-10T20:17:15,791 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/9d842a7067a64ae2ad2908a0a1dd3bbd is 50, key is test_row_0/B:col10/1733861834162/Put/seqid=0 2024-12-10T20:17:15,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742290_1466 (size=12207) 2024-12-10T20:17:15,811 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/9d842a7067a64ae2ad2908a0a1dd3bbd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9d842a7067a64ae2ad2908a0a1dd3bbd 2024-12-10T20:17:15,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742291_1467 (size=12207) 2024-12-10T20:17:15,823 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 9d842a7067a64ae2ad2908a0a1dd3bbd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:15,823 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:15,823 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861835775; duration=0sec 2024-12-10T20:17:15,823 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:15,823 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:15,823 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T20:17:15,826 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:15,826 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/1c72909e98d14372932812b13029fec8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/1c72909e98d14372932812b13029fec8 2024-12-10T20:17:15,826 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:15,826 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:15,827 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:15,827 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/a7be9eedfdd649a782b0374d13c6d13d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8b019d1d4bf74f53825065bbdf87549f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5d0451da506d41cca03702092acef5f9] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=35.3 K 2024-12-10T20:17:15,827 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-10T20:17:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T20:17:15,829 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting a7be9eedfdd649a782b0374d13c6d13d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733861832267 2024-12-10T20:17:15,829 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:15,830 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b019d1d4bf74f53825065bbdf87549f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733861833483 2024-12-10T20:17:15,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:15,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:15,830 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0451da506d41cca03702092acef5f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733861834162 2024-12-10T20:17:15,835 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into 1c72909e98d14372932812b13029fec8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:15,835 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:15,835 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861835775; duration=0sec 2024-12-10T20:17:15,835 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:15,835 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:15,841 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:15,842 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/5e05df6f9b264a968b5e49e69e84198d is 50, key is test_row_0/C:col10/1733861834162/Put/seqid=0 2024-12-10T20:17:15,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742292_1468 (size=12207) 2024-12-10T20:17:15,850 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/5e05df6f9b264a968b5e49e69e84198d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5e05df6f9b264a968b5e49e69e84198d 2024-12-10T20:17:15,854 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 5e05df6f9b264a968b5e49e69e84198d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:15,854 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:15,854 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861835775; duration=0sec 2024-12-10T20:17:15,854 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:15,854 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T20:17:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:15,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:17:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:15,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:15,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T20:17:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:15,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:15,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/6c78707fc0c14675b4587f480eff7cec is 50, key is test_row_0/A:col10/1733861835981/Put/seqid=0 2024-12-10T20:17:15,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742293_1469 (size=12001) 2024-12-10T20:17:15,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/6c78707fc0c14675b4587f480eff7cec 2024-12-10T20:17:15,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ed3372386d744fa988e8cc41ef77c2df is 50, key is test_row_0/B:col10/1733861835981/Put/seqid=0 2024-12-10T20:17:16,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861895995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742294_1470 (size=12001) 2024-12-10T20:17:16,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861896002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861896002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861896003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861896003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861896106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861896106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861896117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861896117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861896117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T20:17:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T20:17:16,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T20:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861896327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861896327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861896336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861896337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861896337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ed3372386d744fa988e8cc41ef77c2df 2024-12-10T20:17:16,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/cad61b015cc248b0be9639543060c6d0 is 50, key is test_row_0/C:col10/1733861835981/Put/seqid=0 2024-12-10T20:17:16,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742295_1471 (size=12001) 2024-12-10T20:17:16,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/cad61b015cc248b0be9639543060c6d0 2024-12-10T20:17:16,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/6c78707fc0c14675b4587f480eff7cec as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6c78707fc0c14675b4587f480eff7cec 2024-12-10T20:17:16,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6c78707fc0c14675b4587f480eff7cec, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T20:17:16,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ed3372386d744fa988e8cc41ef77c2df as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ed3372386d744fa988e8cc41ef77c2df 2024-12-10T20:17:16,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T20:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:16,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:16,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ed3372386d744fa988e8cc41ef77c2df, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T20:17:16,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/cad61b015cc248b0be9639543060c6d0 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/cad61b015cc248b0be9639543060c6d0 2024-12-10T20:17:16,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/cad61b015cc248b0be9639543060c6d0, entries=150, sequenceid=118, filesize=11.7 K 2024-12-10T20:17:16,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 3412e855b2a7dd08270299896873abf4 in 469ms, sequenceid=118, compaction requested=false 2024-12-10T20:17:16,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T20:17:16,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-10T20:17:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:16,596 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:17:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:16,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/6755f194dee64cc480a1eca5e9d576f9 is 50, key is test_row_0/A:col10/1733861836001/Put/seqid=0 2024-12-10T20:17:16,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742296_1472 (size=12101) 2024-12-10T20:17:16,604 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/6755f194dee64cc480a1eca5e9d576f9 2024-12-10T20:17:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/cc155d7c7f224c7d92ab38452ec66501 is 50, key is test_row_0/B:col10/1733861836001/Put/seqid=0 2024-12-10T20:17:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742297_1473 (size=12101) 2024-12-10T20:17:16,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:16,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861896665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861896670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861896671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861896673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861896678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861896787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861896787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861896787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861896796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861896796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T20:17:17,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861896997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861896997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861896997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861897017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861897017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,027 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/cc155d7c7f224c7d92ab38452ec66501 2024-12-10T20:17:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/44af10b247874082b9537676ad790905 is 50, key is test_row_0/C:col10/1733861836001/Put/seqid=0 2024-12-10T20:17:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742298_1474 (size=12101) 2024-12-10T20:17:17,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861897318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861897318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861897318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861897327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861897337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,436 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/44af10b247874082b9537676ad790905 2024-12-10T20:17:17,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/6755f194dee64cc480a1eca5e9d576f9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6755f194dee64cc480a1eca5e9d576f9 2024-12-10T20:17:17,444 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6755f194dee64cc480a1eca5e9d576f9, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T20:17:17,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/cc155d7c7f224c7d92ab38452ec66501 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/cc155d7c7f224c7d92ab38452ec66501 2024-12-10T20:17:17,459 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/cc155d7c7f224c7d92ab38452ec66501, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T20:17:17,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/44af10b247874082b9537676ad790905 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/44af10b247874082b9537676ad790905 2024-12-10T20:17:17,466 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/44af10b247874082b9537676ad790905, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T20:17:17,467 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3412e855b2a7dd08270299896873abf4 in 871ms, sequenceid=131, compaction requested=true 2024-12-10T20:17:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-10T20:17:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-10T20:17:17,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-10T20:17:17,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6380 sec 2024-12-10T20:17:17,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.6430 sec 2024-12-10T20:17:17,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:17:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:17,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T20:17:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:17,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/734ff530e3aa44d6bf1f28e796d4838b is 50, key is test_row_0/A:col10/1733861836672/Put/seqid=0 2024-12-10T20:17:17,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742299_1475 (size=14541) 2024-12-10T20:17:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861897871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861897878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861897885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861897884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861897886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-10T20:17:17,977 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:17,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-10T20:17:17,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T20:17:17,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:17,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:18,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861897996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861897996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861898007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861898007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861898007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T20:17:18,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T20:17:18,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:18,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861898207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861898207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861898216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861898217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861898217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/734ff530e3aa44d6bf1f28e796d4838b 2024-12-10T20:17:18,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/dc0ef5baafcf400a9cac17c472379da6 is 50, key is test_row_0/B:col10/1733861836672/Put/seqid=0 2024-12-10T20:17:18,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742300_1476 (size=12151) 2024-12-10T20:17:18,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/dc0ef5baafcf400a9cac17c472379da6 2024-12-10T20:17:18,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/78160d9e33f04c55bf985e38f5ba9c0f is 50, key is test_row_0/C:col10/1733861836672/Put/seqid=0 2024-12-10T20:17:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742301_1477 (size=12151) 2024-12-10T20:17:18,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T20:17:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T20:17:18,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T20:17:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861898517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861898517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861898536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861898536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861898537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:18,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T20:17:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T20:17:18,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/78160d9e33f04c55bf985e38f5ba9c0f 2024-12-10T20:17:18,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/734ff530e3aa44d6bf1f28e796d4838b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/734ff530e3aa44d6bf1f28e796d4838b 2024-12-10T20:17:18,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/734ff530e3aa44d6bf1f28e796d4838b, entries=200, sequenceid=151, filesize=14.2 K 2024-12-10T20:17:18,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/dc0ef5baafcf400a9cac17c472379da6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dc0ef5baafcf400a9cac17c472379da6 2024-12-10T20:17:18,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dc0ef5baafcf400a9cac17c472379da6, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T20:17:18,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/78160d9e33f04c55bf985e38f5ba9c0f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/78160d9e33f04c55bf985e38f5ba9c0f 2024-12-10T20:17:18,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/78160d9e33f04c55bf985e38f5ba9c0f, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T20:17:18,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3412e855b2a7dd08270299896873abf4 in 873ms, sequenceid=151, compaction requested=true 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:18,704 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:18,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:18,704 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:17:18,705 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:17:18,705 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:17:18,705 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:18,705 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:18,705 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,705 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,705 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/1c72909e98d14372932812b13029fec8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6c78707fc0c14675b4587f480eff7cec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6755f194dee64cc480a1eca5e9d576f9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/734ff530e3aa44d6bf1f28e796d4838b] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=49.7 K 2024-12-10T20:17:18,705 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9d842a7067a64ae2ad2908a0a1dd3bbd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ed3372386d744fa988e8cc41ef77c2df, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/cc155d7c7f224c7d92ab38452ec66501, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dc0ef5baafcf400a9cac17c472379da6] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=47.3 K 2024-12-10T20:17:18,706 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d842a7067a64ae2ad2908a0a1dd3bbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733861834162 2024-12-10T20:17:18,706 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c72909e98d14372932812b13029fec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733861834162 2024-12-10T20:17:18,706 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3372386d744fa988e8cc41ef77c2df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733861835340 2024-12-10T20:17:18,706 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c78707fc0c14675b4587f480eff7cec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733861835340 2024-12-10T20:17:18,706 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6755f194dee64cc480a1eca5e9d576f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733861835993 2024-12-10T20:17:18,706 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting cc155d7c7f224c7d92ab38452ec66501, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733861835993 2024-12-10T20:17:18,707 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734ff530e3aa44d6bf1f28e796d4838b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733861836669 2024-12-10T20:17:18,707 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting dc0ef5baafcf400a9cac17c472379da6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733861836672 2024-12-10T20:17:18,719 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:18,719 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/b47747ee5bb041099e105352ef742929 is 50, key is test_row_0/A:col10/1733861836672/Put/seqid=0 2024-12-10T20:17:18,722 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:18,722 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/15c20b302bb34d08b9ebf9f046742d6d is 50, key is test_row_0/B:col10/1733861836672/Put/seqid=0 2024-12-10T20:17:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742302_1478 (size=12493) 2024-12-10T20:17:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742303_1479 (size=12493) 2024-12-10T20:17:18,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:18,748 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:18,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/706f0f476fa3495697a6a19c31d24aaa is 50, key is test_row_0/A:col10/1733861837883/Put/seqid=0 2024-12-10T20:17:18,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742304_1480 (size=12151) 2024-12-10T20:17:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:19,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861899077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861899078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861899079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861899084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861899087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T20:17:19,138 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/15c20b302bb34d08b9ebf9f046742d6d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/15c20b302bb34d08b9ebf9f046742d6d 2024-12-10T20:17:19,138 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/b47747ee5bb041099e105352ef742929 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/b47747ee5bb041099e105352ef742929 2024-12-10T20:17:19,142 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into b47747ee5bb041099e105352ef742929(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:19,142 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 15c20b302bb34d08b9ebf9f046742d6d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:19,142 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:19,142 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:19,142 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=12, startTime=1733861838704; duration=0sec 2024-12-10T20:17:19,142 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=12, startTime=1733861838704; duration=0sec 2024-12-10T20:17:19,142 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:19,142 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:19,142 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:19,143 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:19,143 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:17:19,143 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:17:19,143 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:19,143 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:19,144 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5e05df6f9b264a968b5e49e69e84198d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/cad61b015cc248b0be9639543060c6d0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/44af10b247874082b9537676ad790905, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/78160d9e33f04c55bf985e38f5ba9c0f] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=47.3 K 2024-12-10T20:17:19,146 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e05df6f9b264a968b5e49e69e84198d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733861834162 2024-12-10T20:17:19,146 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting cad61b015cc248b0be9639543060c6d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733861835340 2024-12-10T20:17:19,146 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 44af10b247874082b9537676ad790905, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733861835993 2024-12-10T20:17:19,146 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 78160d9e33f04c55bf985e38f5ba9c0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733861836672 2024-12-10T20:17:19,153 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:19,154 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/88ca1a9b068740a29158c21f06c5aca8 is 50, key is test_row_0/C:col10/1733861836672/Put/seqid=0 2024-12-10T20:17:19,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742305_1481 (size=12493) 2024-12-10T20:17:19,160 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/706f0f476fa3495697a6a19c31d24aaa 2024-12-10T20:17:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/12ff2f3aec9542cda6af2863f7645d1f is 50, key is test_row_0/B:col10/1733861837883/Put/seqid=0 2024-12-10T20:17:19,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742306_1482 (size=12151) 2024-12-10T20:17:19,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861899197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861899197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861899197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861899197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861899197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T20:17:19,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T20:17:19,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861899407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861899407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861899407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861899407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861899416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,560 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/88ca1a9b068740a29158c21f06c5aca8 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/88ca1a9b068740a29158c21f06c5aca8 2024-12-10T20:17:19,564 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 88ca1a9b068740a29158c21f06c5aca8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:19,564 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:19,564 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=12, startTime=1733861838704; duration=0sec 2024-12-10T20:17:19,564 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:19,564 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:19,568 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/12ff2f3aec9542cda6af2863f7645d1f 2024-12-10T20:17:19,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/4db51b4d303542a3b7bbea1a684c9afe is 50, key is test_row_0/C:col10/1733861837883/Put/seqid=0 2024-12-10T20:17:19,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742307_1483 (size=12151) 2024-12-10T20:17:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861899717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861899717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861899727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861899727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861899727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:19,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:19,977 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/4db51b4d303542a3b7bbea1a684c9afe 2024-12-10T20:17:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/706f0f476fa3495697a6a19c31d24aaa as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/706f0f476fa3495697a6a19c31d24aaa 2024-12-10T20:17:19,998 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/706f0f476fa3495697a6a19c31d24aaa, entries=150, sequenceid=163, filesize=11.9 K 2024-12-10T20:17:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/12ff2f3aec9542cda6af2863f7645d1f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/12ff2f3aec9542cda6af2863f7645d1f 2024-12-10T20:17:20,004 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/12ff2f3aec9542cda6af2863f7645d1f, entries=150, sequenceid=163, filesize=11.9 K 2024-12-10T20:17:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/4db51b4d303542a3b7bbea1a684c9afe as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/4db51b4d303542a3b7bbea1a684c9afe 2024-12-10T20:17:20,008 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/4db51b4d303542a3b7bbea1a684c9afe, entries=150, sequenceid=163, filesize=11.9 K 2024-12-10T20:17:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T20:17:20,010 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3412e855b2a7dd08270299896873abf4 in 1261ms, sequenceid=163, compaction requested=false 2024-12-10T20:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-10T20:17:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-10T20:17:20,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-10T20:17:20,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-12-10T20:17:20,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0350 sec 2024-12-10T20:17:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T20:17:20,137 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:20,137 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-10T20:17:20,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T20:17:20,139 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:20,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:20,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T20:17:20,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:20,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:20,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:20,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:20,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:20,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:20,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/a76e3eb2b3d0462bb2201b728974eecd is 50, key is test_row_0/A:col10/1733861839085/Put/seqid=0 2024-12-10T20:17:20,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742308_1484 (size=14541) 2024-12-10T20:17:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T20:17:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861900247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861900247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861900249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861900255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861900255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861900367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861900367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861900367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T20:17:20,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861900587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861900587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861900587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/a76e3eb2b3d0462bb2201b728974eecd 2024-12-10T20:17:20,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/b3f0f6263edd4c40bea663a1e42584a4 is 50, key is test_row_0/B:col10/1733861839085/Put/seqid=0 2024-12-10T20:17:20,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742309_1485 (size=12151) 2024-12-10T20:17:20,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:20,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T20:17:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861900897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861900898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861900898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:20,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:20,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:21,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/b3f0f6263edd4c40bea663a1e42584a4 2024-12-10T20:17:21,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:21,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:21,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:21,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:21,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/898f542f54f94a8592c0c51b40c2dd94 is 50, key is test_row_0/C:col10/1733861839085/Put/seqid=0 2024-12-10T20:17:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742310_1486 (size=12151) 2024-12-10T20:17:21,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/898f542f54f94a8592c0c51b40c2dd94 2024-12-10T20:17:21,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/a76e3eb2b3d0462bb2201b728974eecd as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a76e3eb2b3d0462bb2201b728974eecd 2024-12-10T20:17:21,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a76e3eb2b3d0462bb2201b728974eecd, entries=200, sequenceid=187, filesize=14.2 K 2024-12-10T20:17:21,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/b3f0f6263edd4c40bea663a1e42584a4 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b3f0f6263edd4c40bea663a1e42584a4 2024-12-10T20:17:21,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b3f0f6263edd4c40bea663a1e42584a4, entries=150, sequenceid=187, filesize=11.9 K 2024-12-10T20:17:21,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/898f542f54f94a8592c0c51b40c2dd94 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/898f542f54f94a8592c0c51b40c2dd94 2024-12-10T20:17:21,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/898f542f54f94a8592c0c51b40c2dd94, entries=150, sequenceid=187, filesize=11.9 K 2024-12-10T20:17:21,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3412e855b2a7dd08270299896873abf4 in 866ms, sequenceid=187, compaction requested=true 2024-12-10T20:17:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:21,098 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:21,098 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:21,098 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:21,098 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:21,099 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:21,099 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/b47747ee5bb041099e105352ef742929, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/706f0f476fa3495697a6a19c31d24aaa, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a76e3eb2b3d0462bb2201b728974eecd] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=38.3 K 2024-12-10T20:17:21,099 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:21,099 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:21,099 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:21,099 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/15c20b302bb34d08b9ebf9f046742d6d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/12ff2f3aec9542cda6af2863f7645d1f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b3f0f6263edd4c40bea663a1e42584a4] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=35.9 K 2024-12-10T20:17:21,099 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c20b302bb34d08b9ebf9f046742d6d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733861836672 2024-12-10T20:17:21,099 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting b47747ee5bb041099e105352ef742929, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733861836672 2024-12-10T20:17:21,100 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 12ff2f3aec9542cda6af2863f7645d1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733861837882 2024-12-10T20:17:21,100 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706f0f476fa3495697a6a19c31d24aaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733861837882 2024-12-10T20:17:21,100 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f0f6263edd4c40bea663a1e42584a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733861839082 2024-12-10T20:17:21,100 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76e3eb2b3d0462bb2201b728974eecd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733861839077 2024-12-10T20:17:21,108 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:21,108 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f872d40d75a742a6961a4ba90ccfb4ba is 50, key is test_row_0/A:col10/1733861839085/Put/seqid=0 2024-12-10T20:17:21,114 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#415 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:21,114 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/490b030edc6d4d709b4822854300d101 is 50, key is test_row_0/B:col10/1733861839085/Put/seqid=0 2024-12-10T20:17:21,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742311_1487 (size=12595) 2024-12-10T20:17:21,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742312_1488 (size=12595) 2024-12-10T20:17:21,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T20:17:21,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:21,213 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T20:17:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f0d4a19686fd4e16ac6ef0b77c642a47 is 50, key is test_row_0/A:col10/1733861840253/Put/seqid=0 2024-12-10T20:17:21,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742313_1489 (size=12151) 2024-12-10T20:17:21,222 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f0d4a19686fd4e16ac6ef0b77c642a47 2024-12-10T20:17:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ac1bb4f6878e431a85289afe456976f6 is 50, key is test_row_0/B:col10/1733861840253/Put/seqid=0 2024-12-10T20:17:21,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742314_1490 (size=12151) 2024-12-10T20:17:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:21,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T20:17:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861901341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861841321, max timestamp=1733861841353Num uniques 859; 2024-12-10T20:17:21,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861841321, max timestamp=1733861841353Num uniques 859; 2024-12-10T20:17:21,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733861841321, max timestamp=1733861841353Num uniques 859; 2024-12-10T20:17:21,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:17:21,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:17:21,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T20:17:21,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861901433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861901434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861901434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861901435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861901457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,518 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f872d40d75a742a6961a4ba90ccfb4ba as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f872d40d75a742a6961a4ba90ccfb4ba 2024-12-10T20:17:21,521 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into f872d40d75a742a6961a4ba90ccfb4ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:21,521 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:21,522 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861841098; duration=0sec 2024-12-10T20:17:21,522 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:21,522 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:21,522 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:21,522 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:21,522 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:21,522 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:21,523 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/88ca1a9b068740a29158c21f06c5aca8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/4db51b4d303542a3b7bbea1a684c9afe, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/898f542f54f94a8592c0c51b40c2dd94] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=35.9 K 2024-12-10T20:17:21,523 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88ca1a9b068740a29158c21f06c5aca8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733861836672 2024-12-10T20:17:21,524 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/490b030edc6d4d709b4822854300d101 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/490b030edc6d4d709b4822854300d101 2024-12-10T20:17:21,524 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db51b4d303542a3b7bbea1a684c9afe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733861837882 2024-12-10T20:17:21,524 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 898f542f54f94a8592c0c51b40c2dd94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733861839082 2024-12-10T20:17:21,527 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 490b030edc6d4d709b4822854300d101(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:21,527 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:21,527 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861841098; duration=0sec 2024-12-10T20:17:21,527 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:21,527 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:21,529 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:21,529 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/53c1c798ab994a3296313b98995e7469 is 50, key is test_row_0/C:col10/1733861839085/Put/seqid=0 2024-12-10T20:17:21,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742315_1491 (size=12595) 2024-12-10T20:17:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861901546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861901546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861901557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861901557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,646 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ac1bb4f6878e431a85289afe456976f6 2024-12-10T20:17:21,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/3621ac27e6d34d0cb3eb1c065f651431 is 50, key is test_row_0/C:col10/1733861840253/Put/seqid=0 2024-12-10T20:17:21,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742316_1492 (size=12151) 2024-12-10T20:17:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861901677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861901757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861901757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861901767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861901767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:21,937 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/53c1c798ab994a3296313b98995e7469 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/53c1c798ab994a3296313b98995e7469 2024-12-10T20:17:21,942 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 53c1c798ab994a3296313b98995e7469(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:21,942 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:21,942 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861841098; duration=0sec 2024-12-10T20:17:21,942 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:21,942 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:22,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861901997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,055 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/3621ac27e6d34d0cb3eb1c065f651431 2024-12-10T20:17:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/f0d4a19686fd4e16ac6ef0b77c642a47 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f0d4a19686fd4e16ac6ef0b77c642a47 2024-12-10T20:17:22,062 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f0d4a19686fd4e16ac6ef0b77c642a47, entries=150, sequenceid=198, filesize=11.9 K 2024-12-10T20:17:22,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ac1bb4f6878e431a85289afe456976f6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac1bb4f6878e431a85289afe456976f6 2024-12-10T20:17:22,069 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac1bb4f6878e431a85289afe456976f6, entries=150, sequenceid=198, filesize=11.9 K 2024-12-10T20:17:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/3621ac27e6d34d0cb3eb1c065f651431 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3621ac27e6d34d0cb3eb1c065f651431 2024-12-10T20:17:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861902067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861902067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,083 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3621ac27e6d34d0cb3eb1c065f651431, entries=150, sequenceid=198, filesize=11.9 K 2024-12-10T20:17:22,084 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3412e855b2a7dd08270299896873abf4 in 871ms, sequenceid=198, compaction requested=false 2024-12-10T20:17:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-10T20:17:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-10T20:17:22,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-10T20:17:22,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-12-10T20:17:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:22,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T20:17:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:22,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9540 sec 2024-12-10T20:17:22,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/319552ed2d7f44b5ad27a171af3dfebf is 50, key is test_row_0/A:col10/1733861841419/Put/seqid=0 2024-12-10T20:17:22,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742317_1493 (size=14541) 2024-12-10T20:17:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861902122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861902123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861902226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861902236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T20:17:22,287 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:22,287 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-10T20:17:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T20:17:22,288 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:22,289 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:22,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T20:17:22,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861902447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861902447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/319552ed2d7f44b5ad27a171af3dfebf 2024-12-10T20:17:22,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861902517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/3aa5a139e63b4bc5a52938b8303be194 is 50, key is test_row_0/B:col10/1733861841419/Put/seqid=0 2024-12-10T20:17:22,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742318_1494 (size=12151) 2024-12-10T20:17:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861902587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861902587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T20:17:22,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861902757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861902758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:22,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:22,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T20:17:22,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/3aa5a139e63b4bc5a52938b8303be194 2024-12-10T20:17:22,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/19f105a6502d4b8093e98d9239ce5e07 is 50, key is test_row_0/C:col10/1733861841419/Put/seqid=0 2024-12-10T20:17:22,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742319_1495 (size=12151) 2024-12-10T20:17:23,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:23,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:23,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:23,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861903277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861903277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/19f105a6502d4b8093e98d9239ce5e07 2024-12-10T20:17:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/319552ed2d7f44b5ad27a171af3dfebf as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/319552ed2d7f44b5ad27a171af3dfebf 2024-12-10T20:17:23,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/319552ed2d7f44b5ad27a171af3dfebf, entries=200, sequenceid=238, filesize=14.2 K 2024-12-10T20:17:23,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/3aa5a139e63b4bc5a52938b8303be194 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3aa5a139e63b4bc5a52938b8303be194 2024-12-10T20:17:23,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3aa5a139e63b4bc5a52938b8303be194, entries=150, sequenceid=238, filesize=11.9 K 2024-12-10T20:17:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/19f105a6502d4b8093e98d9239ce5e07 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/19f105a6502d4b8093e98d9239ce5e07 2024-12-10T20:17:23,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:23,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:23,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:23,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/19f105a6502d4b8093e98d9239ce5e07, entries=150, sequenceid=238, filesize=11.9 K 2024-12-10T20:17:23,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3412e855b2a7dd08270299896873abf4 in 1267ms, sequenceid=238, compaction requested=true 2024-12-10T20:17:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:23,358 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:23,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:23,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:23,359 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:23,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:23,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:23,359 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:23,359 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:23,359 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,359 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f872d40d75a742a6961a4ba90ccfb4ba, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f0d4a19686fd4e16ac6ef0b77c642a47, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/319552ed2d7f44b5ad27a171af3dfebf] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=38.4 K 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f872d40d75a742a6961a4ba90ccfb4ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733861839082 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:23,360 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,360 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/490b030edc6d4d709b4822854300d101, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac1bb4f6878e431a85289afe456976f6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3aa5a139e63b4bc5a52938b8303be194] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.0 K 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d4a19686fd4e16ac6ef0b77c642a47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733861840247 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 490b030edc6d4d709b4822854300d101, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733861839082 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319552ed2d7f44b5ad27a171af3dfebf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733861841419 2024-12-10T20:17:23,360 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ac1bb4f6878e431a85289afe456976f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733861840247 2024-12-10T20:17:23,361 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aa5a139e63b4bc5a52938b8303be194, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733861841419 2024-12-10T20:17:23,368 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:23,368 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/3af713f57e194b7abb9ec33b7dc77196 is 50, key is test_row_0/A:col10/1733861841419/Put/seqid=0 2024-12-10T20:17:23,370 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:23,370 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/0eeb45076bfe4e0284618e52f5dccb10 is 50, key is test_row_0/B:col10/1733861841419/Put/seqid=0 2024-12-10T20:17:23,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742320_1496 (size=12697) 2024-12-10T20:17:23,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742321_1497 (size=12697) 2024-12-10T20:17:23,406 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/3af713f57e194b7abb9ec33b7dc77196 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/3af713f57e194b7abb9ec33b7dc77196 2024-12-10T20:17:23,406 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/0eeb45076bfe4e0284618e52f5dccb10 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0eeb45076bfe4e0284618e52f5dccb10 2024-12-10T20:17:23,412 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into 3af713f57e194b7abb9ec33b7dc77196(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:23,413 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:23,413 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861843358; duration=0sec 2024-12-10T20:17:23,413 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:23,413 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:23,413 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:23,414 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 0eeb45076bfe4e0284618e52f5dccb10(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:23,414 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:23,414 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861843359; duration=0sec 2024-12-10T20:17:23,414 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:23,414 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:23,415 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:23,415 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:23,415 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,415 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/53c1c798ab994a3296313b98995e7469, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3621ac27e6d34d0cb3eb1c065f651431, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/19f105a6502d4b8093e98d9239ce5e07] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.0 K 2024-12-10T20:17:23,415 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c1c798ab994a3296313b98995e7469, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733861839082 2024-12-10T20:17:23,415 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3621ac27e6d34d0cb3eb1c065f651431, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733861840247 2024-12-10T20:17:23,418 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f105a6502d4b8093e98d9239ce5e07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733861841419 2024-12-10T20:17:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T20:17:23,435 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:23,436 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8f5fa0095c5b466996446f9c71652204 is 50, key is test_row_0/C:col10/1733861841419/Put/seqid=0 2024-12-10T20:17:23,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742322_1498 (size=12697) 2024-12-10T20:17:23,478 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8f5fa0095c5b466996446f9c71652204 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8f5fa0095c5b466996446f9c71652204 2024-12-10T20:17:23,483 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 8f5fa0095c5b466996446f9c71652204(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:23,484 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:23,485 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861843359; duration=0sec 2024-12-10T20:17:23,485 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:23,485 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:23,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T20:17:23,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:23,509 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:17:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:23,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/0592d52268a647c587ad42c2f4b9c52e is 50, key is test_row_0/A:col10/1733861842121/Put/seqid=0 2024-12-10T20:17:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:23,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742323_1499 (size=12151) 2024-12-10T20:17:23,547 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/0592d52268a647c587ad42c2f4b9c52e 2024-12-10T20:17:23,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/0e485ca5a43643c8b6dc65bd1da25f1f is 50, key is test_row_0/B:col10/1733861842121/Put/seqid=0 2024-12-10T20:17:23,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742324_1500 (size=12151) 2024-12-10T20:17:23,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861843604, max timestamp=1733861843627Num uniques 859; 2024-12-10T20:17:23,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861843604, max timestamp=1733861843627Num uniques 859; 2024-12-10T20:17:23,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:17:23,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:17:23,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733861843604, max timestamp=1733861843627Num uniques 859; 2024-12-10T20:17:23,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T20:17:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861903667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861903674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861903677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861903787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861903787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861903787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:23,964 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/0e485ca5a43643c8b6dc65bd1da25f1f 2024-12-10T20:17:23,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/63975a00a4a745a0aa09f0aa4d508dec is 50, key is test_row_0/C:col10/1733861842121/Put/seqid=0 2024-12-10T20:17:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742325_1501 (size=12151) 2024-12-10T20:17:24,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861903997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861904006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861904006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861904287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,294 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 3412e855b2a7dd08270299896873abf4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861904297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,304 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 3412e855b2a7dd08270299896873abf4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861904317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861904326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861904326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,374 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/63975a00a4a745a0aa09f0aa4d508dec 2024-12-10T20:17:24,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/0592d52268a647c587ad42c2f4b9c52e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/0592d52268a647c587ad42c2f4b9c52e 2024-12-10T20:17:24,380 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/0592d52268a647c587ad42c2f4b9c52e, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T20:17:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/0e485ca5a43643c8b6dc65bd1da25f1f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0e485ca5a43643c8b6dc65bd1da25f1f 2024-12-10T20:17:24,385 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0e485ca5a43643c8b6dc65bd1da25f1f, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T20:17:24,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/63975a00a4a745a0aa09f0aa4d508dec as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/63975a00a4a745a0aa09f0aa4d508dec 2024-12-10T20:17:24,388 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/63975a00a4a745a0aa09f0aa4d508dec, entries=150, sequenceid=251, filesize=11.9 K 2024-12-10T20:17:24,389 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3412e855b2a7dd08270299896873abf4 in 880ms, sequenceid=251, compaction requested=false 2024-12-10T20:17:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-10T20:17:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-10T20:17:24,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-10T20:17:24,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1010 sec 2024-12-10T20:17:24,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1050 sec 2024-12-10T20:17:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T20:17:24,437 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:24,437 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-10T20:17:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T20:17:24,439 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:24,439 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:24,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T20:17:24,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:24,592 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:24,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/91f39644237444a190faf6aaf6b91d76 is 50, key is test_row_0/A:col10/1733861843656/Put/seqid=0 2024-12-10T20:17:24,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742326_1502 (size=12301) 2024-12-10T20:17:24,629 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/91f39644237444a190faf6aaf6b91d76 2024-12-10T20:17:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/dce532ee4a24495484f54dc4f71232c6 is 50, key is test_row_0/B:col10/1733861843656/Put/seqid=0 2024-12-10T20:17:24,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742327_1503 (size=12301) 2024-12-10T20:17:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T20:17:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:24,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861904860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861904869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861904870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861904977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861904977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:24,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861904986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:24,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,052 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/dce532ee4a24495484f54dc4f71232c6 2024-12-10T20:17:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/ca2459d2bc6b49f39cf8d13984c65816 is 50, key is test_row_0/C:col10/1733861843656/Put/seqid=0 2024-12-10T20:17:25,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742328_1504 (size=12301) 2024-12-10T20:17:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T20:17:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861905187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861905197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861905197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,461 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/ca2459d2bc6b49f39cf8d13984c65816 2024-12-10T20:17:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/91f39644237444a190faf6aaf6b91d76 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/91f39644237444a190faf6aaf6b91d76 2024-12-10T20:17:25,467 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/91f39644237444a190faf6aaf6b91d76, entries=150, sequenceid=287, filesize=12.0 K 2024-12-10T20:17:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/dce532ee4a24495484f54dc4f71232c6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dce532ee4a24495484f54dc4f71232c6 2024-12-10T20:17:25,469 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dce532ee4a24495484f54dc4f71232c6, entries=150, sequenceid=287, filesize=12.0 K 2024-12-10T20:17:25,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/ca2459d2bc6b49f39cf8d13984c65816 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca2459d2bc6b49f39cf8d13984c65816 2024-12-10T20:17:25,474 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca2459d2bc6b49f39cf8d13984c65816, entries=150, sequenceid=287, filesize=12.0 K 2024-12-10T20:17:25,474 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 3412e855b2a7dd08270299896873abf4 in 882ms, sequenceid=287, compaction requested=true 2024-12-10T20:17:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-10T20:17:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-10T20:17:25,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-10T20:17:25,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2024-12-10T20:17:25,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0410 sec 2024-12-10T20:17:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:25,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T20:17:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:25,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/c89ed8c4b197456bb4a7304259fea265 is 50, key is test_row_0/A:col10/1733861844859/Put/seqid=0 2024-12-10T20:17:25,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742329_1505 (size=14741) 2024-12-10T20:17:25,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/c89ed8c4b197456bb4a7304259fea265 2024-12-10T20:17:25,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/f73ff610ca664144a71975e2b112a00a is 50, key is test_row_0/B:col10/1733861844859/Put/seqid=0 2024-12-10T20:17:25,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742330_1506 (size=12301) 2024-12-10T20:17:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861905552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861905556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861905557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T20:17:25,577 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:25,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-10T20:17:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T20:17:25,579 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:25,579 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:25,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861905666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861905667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861905677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T20:17:25,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T20:17:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:25,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861905877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T20:17:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861905877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T20:17:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861905896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:25,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/f73ff610ca664144a71975e2b112a00a 2024-12-10T20:17:25,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/03da6c4af9f140379f9d906f2b539d3c is 50, key is test_row_0/C:col10/1733861844859/Put/seqid=0 2024-12-10T20:17:25,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742331_1507 (size=12301) 2024-12-10T20:17:25,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/03da6c4af9f140379f9d906f2b539d3c 2024-12-10T20:17:25,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/c89ed8c4b197456bb4a7304259fea265 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c89ed8c4b197456bb4a7304259fea265 2024-12-10T20:17:25,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c89ed8c4b197456bb4a7304259fea265, entries=200, sequenceid=299, filesize=14.4 K 2024-12-10T20:17:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/f73ff610ca664144a71975e2b112a00a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f73ff610ca664144a71975e2b112a00a 2024-12-10T20:17:25,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f73ff610ca664144a71975e2b112a00a, entries=150, sequenceid=299, filesize=12.0 K 2024-12-10T20:17:25,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/03da6c4af9f140379f9d906f2b539d3c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/03da6c4af9f140379f9d906f2b539d3c 2024-12-10T20:17:25,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/03da6c4af9f140379f9d906f2b539d3c, entries=150, sequenceid=299, filesize=12.0 K 2024-12-10T20:17:25,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3412e855b2a7dd08270299896873abf4 in 449ms, sequenceid=299, compaction requested=true 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:25,960 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:25,960 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:17:25,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:25,961 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:17:25,961 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51890 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:17:25,961 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:25,961 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:25,961 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,961 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,961 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0eeb45076bfe4e0284618e52f5dccb10, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0e485ca5a43643c8b6dc65bd1da25f1f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dce532ee4a24495484f54dc4f71232c6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f73ff610ca664144a71975e2b112a00a] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=48.3 K 2024-12-10T20:17:25,961 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/3af713f57e194b7abb9ec33b7dc77196, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/0592d52268a647c587ad42c2f4b9c52e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/91f39644237444a190faf6aaf6b91d76, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c89ed8c4b197456bb4a7304259fea265] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=50.7 K 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eeb45076bfe4e0284618e52f5dccb10, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733861841419 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af713f57e194b7abb9ec33b7dc77196, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733861841419 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e485ca5a43643c8b6dc65bd1da25f1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733861842108 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0592d52268a647c587ad42c2f4b9c52e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733861842108 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f39644237444a190faf6aaf6b91d76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733861843656 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting dce532ee4a24495484f54dc4f71232c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733861843656 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89ed8c4b197456bb4a7304259fea265, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733861844859 2024-12-10T20:17:25,962 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f73ff610ca664144a71975e2b112a00a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733861844859 2024-12-10T20:17:25,976 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:25,976 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/b2412147010d4c4e9d9df4f9b89418ac is 50, key is test_row_0/B:col10/1733861844859/Put/seqid=0 2024-12-10T20:17:25,984 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:25,985 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/c0e47772b6cf4e1eb82619f1fb37419b is 50, key is test_row_0/A:col10/1733861844859/Put/seqid=0 2024-12-10T20:17:25,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742332_1508 (size=12983) 2024-12-10T20:17:25,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742333_1509 (size=12983) 2024-12-10T20:17:25,995 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/c0e47772b6cf4e1eb82619f1fb37419b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c0e47772b6cf4e1eb82619f1fb37419b 2024-12-10T20:17:25,998 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into c0e47772b6cf4e1eb82619f1fb37419b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:25,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:25,998 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=12, startTime=1733861845960; duration=0sec 2024-12-10T20:17:25,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:25,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:25,998 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:17:25,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T20:17:25,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:25,999 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:25,999 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8f5fa0095c5b466996446f9c71652204, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/63975a00a4a745a0aa09f0aa4d508dec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca2459d2bc6b49f39cf8d13984c65816, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/03da6c4af9f140379f9d906f2b539d3c] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=48.3 K 2024-12-10T20:17:25,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f5fa0095c5b466996446f9c71652204, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733861841419 2024-12-10T20:17:25,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63975a00a4a745a0aa09f0aa4d508dec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733861842108 2024-12-10T20:17:25,999 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2459d2bc6b49f39cf8d13984c65816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733861843656 2024-12-10T20:17:26,000 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03da6c4af9f140379f9d906f2b539d3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733861844859 2024-12-10T20:17:26,005 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:26,006 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/9de0673ee02c42e2853e96ff24a618c5 is 50, key is test_row_0/C:col10/1733861844859/Put/seqid=0 2024-12-10T20:17:26,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742334_1510 (size=12983) 2024-12-10T20:17:26,014 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/9de0673ee02c42e2853e96ff24a618c5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9de0673ee02c42e2853e96ff24a618c5 2024-12-10T20:17:26,022 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 9de0673ee02c42e2853e96ff24a618c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:26,022 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:26,022 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=12, startTime=1733861845960; duration=0sec 2024-12-10T20:17:26,022 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:26,022 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:26,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:26,037 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:26,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/ed1736c324c14dea8de7d589521b7b8f is 50, key is test_row_0/A:col10/1733861845556/Put/seqid=0 2024-12-10T20:17:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742335_1511 (size=12301) 2024-12-10T20:17:26,050 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/ed1736c324c14dea8de7d589521b7b8f 2024-12-10T20:17:26,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/52760339b93e498faa7f5e0b338b770c is 50, key is test_row_0/B:col10/1733861845556/Put/seqid=0 2024-12-10T20:17:26,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742336_1512 (size=12301) 2024-12-10T20:17:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:26,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T20:17:26,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861906210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861906215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861906216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861906306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,309 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 3412e855b2a7dd08270299896873abf4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861906327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861906327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861906327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,331 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 3412e855b2a7dd08270299896873abf4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861906327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,392 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/b2412147010d4c4e9d9df4f9b89418ac as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b2412147010d4c4e9d9df4f9b89418ac 2024-12-10T20:17:26,396 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into b2412147010d4c4e9d9df4f9b89418ac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:26,396 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:26,396 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=12, startTime=1733861845960; duration=0sec 2024-12-10T20:17:26,397 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:26,397 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/52760339b93e498faa7f5e0b338b770c 2024-12-10T20:17:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8819b93e7491486da8a0a462f1c8b6ed is 50, key is test_row_0/C:col10/1733861845556/Put/seqid=0 2024-12-10T20:17:26,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742337_1513 (size=12301) 2024-12-10T20:17:26,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861906536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861906537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861906537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T20:17:26,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861906846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861906846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861906846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:26,931 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8819b93e7491486da8a0a462f1c8b6ed 2024-12-10T20:17:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/ed1736c324c14dea8de7d589521b7b8f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/ed1736c324c14dea8de7d589521b7b8f 2024-12-10T20:17:26,948 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/ed1736c324c14dea8de7d589521b7b8f, entries=150, sequenceid=321, filesize=12.0 K 2024-12-10T20:17:26,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/52760339b93e498faa7f5e0b338b770c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/52760339b93e498faa7f5e0b338b770c 2024-12-10T20:17:26,957 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/52760339b93e498faa7f5e0b338b770c, entries=150, sequenceid=321, filesize=12.0 K 2024-12-10T20:17:26,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8819b93e7491486da8a0a462f1c8b6ed as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8819b93e7491486da8a0a462f1c8b6ed 2024-12-10T20:17:26,961 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8819b93e7491486da8a0a462f1c8b6ed, entries=150, sequenceid=321, filesize=12.0 K 2024-12-10T20:17:26,963 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3412e855b2a7dd08270299896873abf4 in 926ms, sequenceid=321, compaction requested=false 2024-12-10T20:17:26,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:26,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-10T20:17:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-10T20:17:26,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-10T20:17:26,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3870 sec 2024-12-10T20:17:26,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3920 sec 2024-12-10T20:17:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:27,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:17:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:27,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/e8a781ef86b24366990bfb8c3ff63492 is 50, key is test_row_0/A:col10/1733861847360/Put/seqid=0 2024-12-10T20:17:27,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742338_1514 (size=14741) 2024-12-10T20:17:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861907404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861907414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861907415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861907517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861907527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861907527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T20:17:27,729 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:27,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-10T20:17:27,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T20:17:27,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:27,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861907737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861907747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861907748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:27,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/e8a781ef86b24366990bfb8c3ff63492 2024-12-10T20:17:27,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/f4e6c070ef5946cfa2844e2c03a5f7d6 is 50, key is test_row_0/B:col10/1733861847360/Put/seqid=0 2024-12-10T20:17:27,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742339_1515 (size=12301) 2024-12-10T20:17:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T20:17:27,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T20:17:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:27,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T20:17:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T20:17:28,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861908047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861908057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861908057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/f4e6c070ef5946cfa2844e2c03a5f7d6 2024-12-10T20:17:28,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T20:17:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:28,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/503b6f4f4a1d41848b89af2f7d211e9d is 50, key is test_row_0/C:col10/1733861847360/Put/seqid=0 2024-12-10T20:17:28,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742340_1516 (size=12301) 2024-12-10T20:17:28,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/503b6f4f4a1d41848b89af2f7d211e9d 2024-12-10T20:17:28,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/e8a781ef86b24366990bfb8c3ff63492 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e8a781ef86b24366990bfb8c3ff63492 2024-12-10T20:17:28,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e8a781ef86b24366990bfb8c3ff63492, entries=200, sequenceid=335, filesize=14.4 K 2024-12-10T20:17:28,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/f4e6c070ef5946cfa2844e2c03a5f7d6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f4e6c070ef5946cfa2844e2c03a5f7d6 2024-12-10T20:17:28,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f4e6c070ef5946cfa2844e2c03a5f7d6, entries=150, sequenceid=335, filesize=12.0 K 2024-12-10T20:17:28,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/503b6f4f4a1d41848b89af2f7d211e9d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/503b6f4f4a1d41848b89af2f7d211e9d 2024-12-10T20:17:28,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/503b6f4f4a1d41848b89af2f7d211e9d, entries=150, sequenceid=335, filesize=12.0 K 2024-12-10T20:17:28,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 3412e855b2a7dd08270299896873abf4 in 861ms, sequenceid=335, compaction requested=true 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:28,223 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:28,223 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:28,224 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:28,224 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:28,224 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:28,224 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:28,224 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,224 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,224 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b2412147010d4c4e9d9df4f9b89418ac, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/52760339b93e498faa7f5e0b338b770c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f4e6c070ef5946cfa2844e2c03a5f7d6] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.7 K 2024-12-10T20:17:28,224 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c0e47772b6cf4e1eb82619f1fb37419b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/ed1736c324c14dea8de7d589521b7b8f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e8a781ef86b24366990bfb8c3ff63492] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=39.1 K 2024-12-10T20:17:28,224 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting b2412147010d4c4e9d9df4f9b89418ac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733861844859 2024-12-10T20:17:28,225 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e47772b6cf4e1eb82619f1fb37419b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733861844859 2024-12-10T20:17:28,225 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 52760339b93e498faa7f5e0b338b770c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733861845551 2024-12-10T20:17:28,227 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e6c070ef5946cfa2844e2c03a5f7d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733861846198 2024-12-10T20:17:28,227 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1736c324c14dea8de7d589521b7b8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733861845551 2024-12-10T20:17:28,227 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a781ef86b24366990bfb8c3ff63492, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733861846198 2024-12-10T20:17:28,238 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:28,238 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/4d8555c751444313b13cd0e84ee9846e is 50, key is test_row_0/B:col10/1733861847360/Put/seqid=0 2024-12-10T20:17:28,241 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:28,241 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/844cca8b84e6473c8d53b795d4d70496 is 50, key is test_row_0/A:col10/1733861847360/Put/seqid=0 2024-12-10T20:17:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742341_1517 (size=13085) 2024-12-10T20:17:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742342_1518 (size=13085) 2024-12-10T20:17:28,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,345 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:28,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/4c084e1c49654eb59e9c1da35b590e09 is 50, key is test_row_0/A:col10/1733861847402/Put/seqid=0 2024-12-10T20:17:28,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742343_1519 (size=12301) 2024-12-10T20:17:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T20:17:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:28,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:28,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861908589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861908593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861908603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,658 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/4d8555c751444313b13cd0e84ee9846e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4d8555c751444313b13cd0e84ee9846e 2024-12-10T20:17:28,660 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/844cca8b84e6473c8d53b795d4d70496 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/844cca8b84e6473c8d53b795d4d70496 2024-12-10T20:17:28,662 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 4d8555c751444313b13cd0e84ee9846e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:28,662 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:28,662 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861848223; duration=0sec 2024-12-10T20:17:28,662 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:28,662 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:28,662 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:28,663 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:28,663 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:28,663 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,663 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9de0673ee02c42e2853e96ff24a618c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8819b93e7491486da8a0a462f1c8b6ed, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/503b6f4f4a1d41848b89af2f7d211e9d] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.7 K 2024-12-10T20:17:28,664 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de0673ee02c42e2853e96ff24a618c5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733861844859 2024-12-10T20:17:28,664 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into 844cca8b84e6473c8d53b795d4d70496(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:28,664 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:28,664 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861848223; duration=0sec 2024-12-10T20:17:28,664 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8819b93e7491486da8a0a462f1c8b6ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733861845551 2024-12-10T20:17:28,664 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:28,664 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:28,664 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 503b6f4f4a1d41848b89af2f7d211e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733861846198 2024-12-10T20:17:28,671 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:28,671 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/0c2146681f934a92b2dd7ce12d62ab66 is 50, key is test_row_0/C:col10/1733861847360/Put/seqid=0 2024-12-10T20:17:28,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742344_1520 (size=13085) 2024-12-10T20:17:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861908707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861908707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861908727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,753 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/4c084e1c49654eb59e9c1da35b590e09 2024-12-10T20:17:28,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/514287192e33483280c383882431927a is 50, key is test_row_0/B:col10/1733861847402/Put/seqid=0 2024-12-10T20:17:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742345_1521 (size=12301) 2024-12-10T20:17:28,768 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/514287192e33483280c383882431927a 2024-12-10T20:17:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/ca5802ebb212406fa80a80689c071e86 is 50, key is test_row_0/C:col10/1733861847402/Put/seqid=0 2024-12-10T20:17:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742346_1522 (size=12301) 2024-12-10T20:17:28,786 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/ca5802ebb212406fa80a80689c071e86 2024-12-10T20:17:28,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/4c084e1c49654eb59e9c1da35b590e09 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/4c084e1c49654eb59e9c1da35b590e09 2024-12-10T20:17:28,792 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/4c084e1c49654eb59e9c1da35b590e09, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T20:17:28,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/514287192e33483280c383882431927a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/514287192e33483280c383882431927a 2024-12-10T20:17:28,795 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/514287192e33483280c383882431927a, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T20:17:28,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/ca5802ebb212406fa80a80689c071e86 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca5802ebb212406fa80a80689c071e86 2024-12-10T20:17:28,801 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca5802ebb212406fa80a80689c071e86, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T20:17:28,802 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3412e855b2a7dd08270299896873abf4 in 457ms, sequenceid=355, compaction requested=false 2024-12-10T20:17:28,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:28,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:28,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-10T20:17:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-10T20:17:28,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-10T20:17:28,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-12-10T20:17:28,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0750 sec 2024-12-10T20:17:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T20:17:28,867 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:28,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-10T20:17:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T20:17:28,869 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:28,870 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:28,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:28,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T20:17:28,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:28,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:28,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:28,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:28,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:28,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:28,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/9f27648f484d49a7a42d3ac923f09bd3 is 50, key is test_row_0/A:col10/1733861848592/Put/seqid=0 2024-12-10T20:17:28,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742347_1523 (size=14741) 2024-12-10T20:17:28,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/9f27648f484d49a7a42d3ac923f09bd3 2024-12-10T20:17:28,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/4bd5eb5351ad45418b63c31ebc38c7de is 50, key is test_row_0/B:col10/1733861848592/Put/seqid=0 2024-12-10T20:17:28,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742348_1524 (size=12301) 2024-12-10T20:17:28,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861908964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T20:17:28,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861908974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:28,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861908975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T20:17:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,082 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/0c2146681f934a92b2dd7ce12d62ab66 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/0c2146681f934a92b2dd7ce12d62ab66 2024-12-10T20:17:29,087 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 0c2146681f934a92b2dd7ce12d62ab66(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:29,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:29,087 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861848223; duration=0sec 2024-12-10T20:17:29,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:29,087 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861909087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861909087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861909088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T20:17:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T20:17:29,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861909296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861909297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861909297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T20:17:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/4bd5eb5351ad45418b63c31ebc38c7de 2024-12-10T20:17:29,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/07635484a99b4f3fbc66d5aa3689b11b is 50, key is test_row_0/C:col10/1733861848592/Put/seqid=0 2024-12-10T20:17:29,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742349_1525 (size=12301) 2024-12-10T20:17:29,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T20:17:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T20:17:29,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861909607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861909607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861909608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:29,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T20:17:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:29,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/07635484a99b4f3fbc66d5aa3689b11b 2024-12-10T20:17:29,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/9f27648f484d49a7a42d3ac923f09bd3 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/9f27648f484d49a7a42d3ac923f09bd3 2024-12-10T20:17:29,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/9f27648f484d49a7a42d3ac923f09bd3, entries=200, sequenceid=370, filesize=14.4 K 2024-12-10T20:17:29,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/4bd5eb5351ad45418b63c31ebc38c7de as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4bd5eb5351ad45418b63c31ebc38c7de 2024-12-10T20:17:29,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4bd5eb5351ad45418b63c31ebc38c7de, entries=150, sequenceid=370, filesize=12.0 K 2024-12-10T20:17:29,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/07635484a99b4f3fbc66d5aa3689b11b as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/07635484a99b4f3fbc66d5aa3689b11b 2024-12-10T20:17:29,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/07635484a99b4f3fbc66d5aa3689b11b, entries=150, sequenceid=370, filesize=12.0 K 2024-12-10T20:17:29,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3412e855b2a7dd08270299896873abf4 in 862ms, sequenceid=370, compaction requested=true 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:29,781 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:29,781 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:29,782 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,782 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4d8555c751444313b13cd0e84ee9846e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/514287192e33483280c383882431927a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4bd5eb5351ad45418b63c31ebc38c7de] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.8 K 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8555c751444313b13cd0e84ee9846e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733861846198 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 514287192e33483280c383882431927a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733861847402 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd5eb5351ad45418b63c31ebc38c7de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733861848592 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:29,782 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:29,782 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,783 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/844cca8b84e6473c8d53b795d4d70496, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/4c084e1c49654eb59e9c1da35b590e09, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/9f27648f484d49a7a42d3ac923f09bd3] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=39.2 K 2024-12-10T20:17:29,783 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844cca8b84e6473c8d53b795d4d70496, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733861846198 2024-12-10T20:17:29,783 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c084e1c49654eb59e9c1da35b590e09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733861847402 2024-12-10T20:17:29,783 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f27648f484d49a7a42d3ac923f09bd3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733861848580 2024-12-10T20:17:29,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:29,785 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/bb15d7b8780b4ebebd5a82027a7d43b6 is 50, key is test_row_0/A:col10/1733861848973/Put/seqid=0 2024-12-10T20:17:29,790 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#454 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:29,790 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/027e8755bfbb4f58bc8e99e06fd85c68 is 50, key is test_row_0/B:col10/1733861848592/Put/seqid=0 2024-12-10T20:17:29,792 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:29,792 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/32391af8d1e24792a90d8f4d55099b8a is 50, key is test_row_0/A:col10/1733861848592/Put/seqid=0 2024-12-10T20:17:29,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742350_1526 (size=12301) 2024-12-10T20:17:29,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742351_1527 (size=13187) 2024-12-10T20:17:29,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742352_1528 (size=13187) 2024-12-10T20:17:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T20:17:30,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:30,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861910146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861910147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861910151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,194 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/bb15d7b8780b4ebebd5a82027a7d43b6 2024-12-10T20:17:30,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/9c5eea3b87d542b8bff8e0212788c1c6 is 50, key is test_row_0/B:col10/1733861848973/Put/seqid=0 2024-12-10T20:17:30,203 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/027e8755bfbb4f58bc8e99e06fd85c68 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/027e8755bfbb4f58bc8e99e06fd85c68 2024-12-10T20:17:30,208 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/32391af8d1e24792a90d8f4d55099b8a as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/32391af8d1e24792a90d8f4d55099b8a 2024-12-10T20:17:30,212 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into 32391af8d1e24792a90d8f4d55099b8a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:30,212 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into 027e8755bfbb4f58bc8e99e06fd85c68(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:30,212 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861849781; duration=0sec 2024-12-10T20:17:30,212 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861849781; duration=0sec 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:30,212 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:30,213 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:30,213 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:30,213 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:30,213 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/0c2146681f934a92b2dd7ce12d62ab66, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca5802ebb212406fa80a80689c071e86, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/07635484a99b4f3fbc66d5aa3689b11b] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.8 K 2024-12-10T20:17:30,213 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2146681f934a92b2dd7ce12d62ab66, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733861846198 2024-12-10T20:17:30,213 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5802ebb212406fa80a80689c071e86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733861847402 2024-12-10T20:17:30,214 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07635484a99b4f3fbc66d5aa3689b11b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733861848592 2024-12-10T20:17:30,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742353_1529 (size=12301) 2024-12-10T20:17:30,221 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/9c5eea3b87d542b8bff8e0212788c1c6 2024-12-10T20:17:30,223 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:30,223 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/9deef7fe43094e7b989f01594b4fb7e5 is 50, key is test_row_0/C:col10/1733861848592/Put/seqid=0 2024-12-10T20:17:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8549226d38d74180ada5bace4a38e20f is 50, key is test_row_0/C:col10/1733861848973/Put/seqid=0 2024-12-10T20:17:30,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742354_1530 (size=13187) 2024-12-10T20:17:30,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742355_1531 (size=12301) 2024-12-10T20:17:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861910257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861910257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861910267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38168 deadline: 1733861910357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,366 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 3412e855b2a7dd08270299896873abf4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38220 deadline: 1733861910367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,375 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 3412e855b2a7dd08270299896873abf4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861910467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861910467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861910477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,645 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/9deef7fe43094e7b989f01594b4fb7e5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9deef7fe43094e7b989f01594b4fb7e5 2024-12-10T20:17:30,647 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8549226d38d74180ada5bace4a38e20f 2024-12-10T20:17:30,648 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 9deef7fe43094e7b989f01594b4fb7e5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:30,648 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:30,648 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861849781; duration=0sec 2024-12-10T20:17:30,648 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:30,648 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:30,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/bb15d7b8780b4ebebd5a82027a7d43b6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/bb15d7b8780b4ebebd5a82027a7d43b6 2024-12-10T20:17:30,652 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/bb15d7b8780b4ebebd5a82027a7d43b6, entries=150, sequenceid=390, filesize=12.0 K 2024-12-10T20:17:30,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/9c5eea3b87d542b8bff8e0212788c1c6 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9c5eea3b87d542b8bff8e0212788c1c6 2024-12-10T20:17:30,655 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9c5eea3b87d542b8bff8e0212788c1c6, entries=150, sequenceid=390, filesize=12.0 K 2024-12-10T20:17:30,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/8549226d38d74180ada5bace4a38e20f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8549226d38d74180ada5bace4a38e20f 2024-12-10T20:17:30,658 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8549226d38d74180ada5bace4a38e20f, entries=150, sequenceid=390, filesize=12.0 K 2024-12-10T20:17:30,659 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3412e855b2a7dd08270299896873abf4 in 874ms, sequenceid=390, compaction requested=false 2024-12-10T20:17:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-10T20:17:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-10T20:17:30,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-10T20:17:30,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7900 sec 2024-12-10T20:17:30,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7940 sec 2024-12-10T20:17:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:30,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T20:17:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:30,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/fd6d152940924034aebe621b073fca20 is 50, key is test_row_0/A:col10/1733861850781/Put/seqid=0 2024-12-10T20:17:30,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742356_1532 (size=17181) 2024-12-10T20:17:30,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861910810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861910813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861910814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861910927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861910927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861910937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T20:17:31,016 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:31,017 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-10T20:17:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T20:17:31,018 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:31,019 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:31,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T20:17:31,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861911136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861911136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861911147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/fd6d152940924034aebe621b073fca20 2024-12-10T20:17:31,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ddb686ed1d074a97b046c4b4bb302044 is 50, key is test_row_0/B:col10/1733861850781/Put/seqid=0 2024-12-10T20:17:31,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742357_1533 (size=12301) 2024-12-10T20:17:31,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T20:17:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861911447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861911447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861911457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,515 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-12-10T20:17:31,515 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:31,515 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,515 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:31,520 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-12-10T20:17:31,520 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:31,521 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,521 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:31,521 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-12-10T20:17:31,521 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:31,521 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,522 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:31,523 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-12-10T20:17:31,523 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:31,523 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,523 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:31,527 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-12-10T20:17:31,527 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:31,527 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,527 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:31,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:31,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ddb686ed1d074a97b046c4b4bb302044 2024-12-10T20:17:31,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/43eefebf1b9e471c96fabd1ce717ee5d is 50, key is test_row_0/C:col10/1733861850781/Put/seqid=0 2024-12-10T20:17:31,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742358_1534 (size=12301) 2024-12-10T20:17:31,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:31,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T20:17:31,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. as already flushing 2024-12-10T20:17:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:31,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38218 deadline: 1733861911967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38174 deadline: 1733861911967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:31,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38194 deadline: 1733861911977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 , the old value is region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:31,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., hostname=03a2ae042c95,38133,1733861689839, seqNum=2 because the exception is null or not the one we care about 2024-12-10T20:17:32,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/43eefebf1b9e471c96fabd1ce717ee5d 2024-12-10T20:17:32,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/fd6d152940924034aebe621b073fca20 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fd6d152940924034aebe621b073fca20 2024-12-10T20:17:32,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fd6d152940924034aebe621b073fca20, entries=250, sequenceid=408, filesize=16.8 K 2024-12-10T20:17:32,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ddb686ed1d074a97b046c4b4bb302044 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ddb686ed1d074a97b046c4b4bb302044 2024-12-10T20:17:32,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ddb686ed1d074a97b046c4b4bb302044, entries=150, sequenceid=408, filesize=12.0 K 2024-12-10T20:17:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/43eefebf1b9e471c96fabd1ce717ee5d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/43eefebf1b9e471c96fabd1ce717ee5d 2024-12-10T20:17:32,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/43eefebf1b9e471c96fabd1ce717ee5d, entries=150, sequenceid=408, filesize=12.0 K 2024-12-10T20:17:32,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 3412e855b2a7dd08270299896873abf4 in 1249ms, sequenceid=408, compaction requested=true 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:17:32,031 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:32,031 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3412e855b2a7dd08270299896873abf4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:17:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/A is initiating minor compaction (all files) 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:32,032 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/A in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/B is initiating minor compaction (all files) 2024-12-10T20:17:32,032 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/B in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:32,032 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/32391af8d1e24792a90d8f4d55099b8a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/bb15d7b8780b4ebebd5a82027a7d43b6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fd6d152940924034aebe621b073fca20] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=41.7 K 2024-12-10T20:17:32,032 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/027e8755bfbb4f58bc8e99e06fd85c68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9c5eea3b87d542b8bff8e0212788c1c6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ddb686ed1d074a97b046c4b4bb302044] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.9 K 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 027e8755bfbb4f58bc8e99e06fd85c68, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733861848592 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32391af8d1e24792a90d8f4d55099b8a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733861848592 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5eea3b87d542b8bff8e0212788c1c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733861848951 2024-12-10T20:17:32,032 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb15d7b8780b4ebebd5a82027a7d43b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733861848951 2024-12-10T20:17:32,033 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting ddb686ed1d074a97b046c4b4bb302044, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733861850137 2024-12-10T20:17:32,033 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6d152940924034aebe621b073fca20, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733861850137 2024-12-10T20:17:32,040 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:32,041 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/d3ded8c38ad844edaaae28c289fcd465 is 50, key is test_row_0/B:col10/1733861850781/Put/seqid=0 2024-12-10T20:17:32,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742359_1535 (size=13289) 2024-12-10T20:17:32,047 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#A#compaction#463 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:32,048 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/dff612f37c7745429434794a22fc6311 is 50, key is test_row_0/A:col10/1733861850781/Put/seqid=0 2024-12-10T20:17:32,054 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/d3ded8c38ad844edaaae28c289fcd465 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d3ded8c38ad844edaaae28c289fcd465 2024-12-10T20:17:32,059 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/B of 3412e855b2a7dd08270299896873abf4 into d3ded8c38ad844edaaae28c289fcd465(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:32,059 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:32,059 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/B, priority=13, startTime=1733861852031; duration=0sec 2024-12-10T20:17:32,059 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:17:32,059 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:B 2024-12-10T20:17:32,059 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:17:32,060 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:17:32,060 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 3412e855b2a7dd08270299896873abf4/C is initiating minor compaction (all files) 2024-12-10T20:17:32,060 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3412e855b2a7dd08270299896873abf4/C in TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:32,060 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9deef7fe43094e7b989f01594b4fb7e5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8549226d38d74180ada5bace4a38e20f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/43eefebf1b9e471c96fabd1ce717ee5d] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp, totalSize=36.9 K 2024-12-10T20:17:32,061 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 9deef7fe43094e7b989f01594b4fb7e5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733861848592 2024-12-10T20:17:32,061 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 8549226d38d74180ada5bace4a38e20f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733861848951 2024-12-10T20:17:32,062 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 43eefebf1b9e471c96fabd1ce717ee5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733861850137 2024-12-10T20:17:32,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T20:17:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:32,088 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:17:32,088 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3412e855b2a7dd08270299896873abf4#C#compaction#464 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:17:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:32,089 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/3aa710fda8a849d693accb897822cc1f is 50, key is test_row_0/C:col10/1733861850781/Put/seqid=0 2024-12-10T20:17:32,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742360_1536 (size=13289) 2024-12-10T20:17:32,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/d754ff86006f443a93e9832f4af2a6ad is 50, key is test_row_0/A:col10/1733861850812/Put/seqid=0 2024-12-10T20:17:32,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742361_1537 (size=13289) 2024-12-10T20:17:32,123 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/3aa710fda8a849d693accb897822cc1f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3aa710fda8a849d693accb897822cc1f 2024-12-10T20:17:32,133 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/C of 3412e855b2a7dd08270299896873abf4 into 3aa710fda8a849d693accb897822cc1f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:32,133 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:32,133 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/C, priority=13, startTime=1733861852031; duration=0sec 2024-12-10T20:17:32,133 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:32,133 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:C 2024-12-10T20:17:32,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742362_1538 (size=12301) 2024-12-10T20:17:32,139 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/d754ff86006f443a93e9832f4af2a6ad 2024-12-10T20:17:32,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ac8c973773434d17942b4dbcb46d6f44 is 50, key is test_row_0/B:col10/1733861850812/Put/seqid=0 2024-12-10T20:17:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T20:17:32,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742363_1539 (size=12301) 2024-12-10T20:17:32,159 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ac8c973773434d17942b4dbcb46d6f44 2024-12-10T20:17:32,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/fe2dda13febb4308818b7b591c326ac7 is 50, key is test_row_0/C:col10/1733861850812/Put/seqid=0 2024-12-10T20:17:32,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742364_1540 (size=12301) 2024-12-10T20:17:32,504 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/dff612f37c7745429434794a22fc6311 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/dff612f37c7745429434794a22fc6311 2024-12-10T20:17:32,511 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3412e855b2a7dd08270299896873abf4/A of 3412e855b2a7dd08270299896873abf4 into dff612f37c7745429434794a22fc6311(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:17:32,511 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:32,511 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., storeName=3412e855b2a7dd08270299896873abf4/A, priority=13, startTime=1733861852031; duration=0sec 2024-12-10T20:17:32,511 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:17:32,511 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3412e855b2a7dd08270299896873abf4:A 2024-12-10T20:17:32,570 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/fe2dda13febb4308818b7b591c326ac7 2024-12-10T20:17:32,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/d754ff86006f443a93e9832f4af2a6ad as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/d754ff86006f443a93e9832f4af2a6ad 2024-12-10T20:17:32,580 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/d754ff86006f443a93e9832f4af2a6ad, entries=150, sequenceid=426, filesize=12.0 K 2024-12-10T20:17:32,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/ac8c973773434d17942b4dbcb46d6f44 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac8c973773434d17942b4dbcb46d6f44 2024-12-10T20:17:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac8c973773434d17942b4dbcb46d6f44, entries=150, sequenceid=426, filesize=12.0 K 2024-12-10T20:17:32,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/fe2dda13febb4308818b7b591c326ac7 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/fe2dda13febb4308818b7b591c326ac7 2024-12-10T20:17:32,591 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/fe2dda13febb4308818b7b591c326ac7, entries=150, sequenceid=426, filesize=12.0 K 2024-12-10T20:17:32,592 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 3412e855b2a7dd08270299896873abf4 in 504ms, sequenceid=426, compaction requested=false 2024-12-10T20:17:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 3412e855b2a7dd08270299896873abf4: 2024-12-10T20:17:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-10T20:17:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-10T20:17:32,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-10T20:17:32,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-12-10T20:17:32,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.5780 sec 2024-12-10T20:17:32,978 INFO [Thread-2023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2023. 2024-12-10T20:17:32,978 INFO [Thread-2015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2015. 2024-12-10T20:17:32,978 DEBUG [Thread-2015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:32,978 DEBUG [Thread-2023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:32,978 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:32,978 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:32,979 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:32,979 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:32,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:32,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:32,988 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-12-10T20:17:32,988 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:32,988 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:32,989 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:32,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T20:17:33,167 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:40,418 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-12-10T20:17:40,419 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:40,419 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:40,419 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:40,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:40,439 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-12-10T20:17:40,439 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:17:40,439 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:40,439 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:40,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1353 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4059 rows 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1365 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4095 rows 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1362 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4086 rows 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1376 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4122 rows 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1368 2024-12-10T20:17:40,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4103 rows 2024-12-10T20:17:40,439 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:17:40,439 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:17:40,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:40,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:40,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:17:40,440 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T20:17:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T20:17:40,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861860442"}]},"ts":"1733861860442"} 2024-12-10T20:17:40,443 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T20:17:40,444 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T20:17:40,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:17:40,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, UNASSIGN}] 2024-12-10T20:17:40,445 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, UNASSIGN 2024-12-10T20:17:40,446 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=3412e855b2a7dd08270299896873abf4, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:40,447 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45643 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=03a2ae042c95,38133,1733861689839, table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-10T20:17:40,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, UNASSIGN because future has completed 2024-12-10T20:17:40,447 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:17:40,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:17:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T20:17:40,600 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 3412e855b2a7dd08270299896873abf4, disabling compactions & flushes 2024-12-10T20:17:40,600 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. after waiting 0 ms 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:40,600 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 3412e855b2a7dd08270299896873abf4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=A 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=B 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3412e855b2a7dd08270299896873abf4, store=C 2024-12-10T20:17:40,600 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:40,605 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/2a94371e42d646cf982e0abdc856d96e is 50, key is test_row_1/A:col10/1733861860417/Put/seqid=0 2024-12-10T20:17:40,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742365_1541 (size=9857) 2024-12-10T20:17:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T20:17:41,013 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/2a94371e42d646cf982e0abdc856d96e 2024-12-10T20:17:41,020 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/3cce43ffce3b45b4b4cd4c64cae38b8f is 50, key is test_row_1/B:col10/1733861860417/Put/seqid=0 2024-12-10T20:17:41,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742366_1542 (size=9857) 2024-12-10T20:17:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T20:17:41,425 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/3cce43ffce3b45b4b4cd4c64cae38b8f 2024-12-10T20:17:41,438 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/260ec70cea4644b281ad217bb09b35df is 50, key is test_row_1/C:col10/1733861860417/Put/seqid=0 2024-12-10T20:17:41,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742367_1543 (size=9857) 2024-12-10T20:17:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T20:17:41,847 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/260ec70cea4644b281ad217bb09b35df 2024-12-10T20:17:41,852 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/A/2a94371e42d646cf982e0abdc856d96e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/2a94371e42d646cf982e0abdc856d96e 2024-12-10T20:17:41,858 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/2a94371e42d646cf982e0abdc856d96e, entries=100, sequenceid=436, filesize=9.6 K 2024-12-10T20:17:41,859 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/B/3cce43ffce3b45b4b4cd4c64cae38b8f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3cce43ffce3b45b4b4cd4c64cae38b8f 2024-12-10T20:17:41,866 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3cce43ffce3b45b4b4cd4c64cae38b8f, entries=100, sequenceid=436, filesize=9.6 K 2024-12-10T20:17:41,866 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/.tmp/C/260ec70cea4644b281ad217bb09b35df as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/260ec70cea4644b281ad217bb09b35df 2024-12-10T20:17:41,871 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/260ec70cea4644b281ad217bb09b35df, entries=100, sequenceid=436, filesize=9.6 K 2024-12-10T20:17:41,872 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3412e855b2a7dd08270299896873abf4 in 1272ms, sequenceid=436, compaction requested=true 2024-12-10T20:17:41,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e5b01d7385354631b9c38507e086c8d2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/de1ddbe5927e44ef9b85c82aa258981d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f51a7fbe126a49718927ede816fc7abc, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a468d96a903c4d65895a7ee90f544f79, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/7bce20a35f874e57aede8090c41f24d1, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fdf68235157b4a0e90bc8ddce9d56aac, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/1c72909e98d14372932812b13029fec8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6c78707fc0c14675b4587f480eff7cec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6755f194dee64cc480a1eca5e9d576f9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/734ff530e3aa44d6bf1f28e796d4838b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/b47747ee5bb041099e105352ef742929, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/706f0f476fa3495697a6a19c31d24aaa, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a76e3eb2b3d0462bb2201b728974eecd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f872d40d75a742a6961a4ba90ccfb4ba, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f0d4a19686fd4e16ac6ef0b77c642a47, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/319552ed2d7f44b5ad27a171af3dfebf, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/3af713f57e194b7abb9ec33b7dc77196, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/0592d52268a647c587ad42c2f4b9c52e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/91f39644237444a190faf6aaf6b91d76, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c89ed8c4b197456bb4a7304259fea265, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c0e47772b6cf4e1eb82619f1fb37419b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/ed1736c324c14dea8de7d589521b7b8f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e8a781ef86b24366990bfb8c3ff63492, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/844cca8b84e6473c8d53b795d4d70496, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/4c084e1c49654eb59e9c1da35b590e09, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/9f27648f484d49a7a42d3ac923f09bd3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/32391af8d1e24792a90d8f4d55099b8a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/bb15d7b8780b4ebebd5a82027a7d43b6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fd6d152940924034aebe621b073fca20] to archive 2024-12-10T20:17:41,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:17:41,882 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/de1ddbe5927e44ef9b85c82aa258981d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/de1ddbe5927e44ef9b85c82aa258981d 2024-12-10T20:17:41,883 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f51a7fbe126a49718927ede816fc7abc to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f51a7fbe126a49718927ede816fc7abc 2024-12-10T20:17:41,884 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e5b01d7385354631b9c38507e086c8d2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e5b01d7385354631b9c38507e086c8d2 2024-12-10T20:17:41,884 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a468d96a903c4d65895a7ee90f544f79 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a468d96a903c4d65895a7ee90f544f79 2024-12-10T20:17:41,886 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/7bce20a35f874e57aede8090c41f24d1 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/7bce20a35f874e57aede8090c41f24d1 2024-12-10T20:17:41,886 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fdf68235157b4a0e90bc8ddce9d56aac to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fdf68235157b4a0e90bc8ddce9d56aac 2024-12-10T20:17:41,892 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6755f194dee64cc480a1eca5e9d576f9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6755f194dee64cc480a1eca5e9d576f9 2024-12-10T20:17:41,892 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/706f0f476fa3495697a6a19c31d24aaa to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/706f0f476fa3495697a6a19c31d24aaa 2024-12-10T20:17:41,892 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/1c72909e98d14372932812b13029fec8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/1c72909e98d14372932812b13029fec8 2024-12-10T20:17:41,892 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6c78707fc0c14675b4587f480eff7cec to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/6c78707fc0c14675b4587f480eff7cec 2024-12-10T20:17:41,893 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f872d40d75a742a6961a4ba90ccfb4ba to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f872d40d75a742a6961a4ba90ccfb4ba 2024-12-10T20:17:41,894 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/734ff530e3aa44d6bf1f28e796d4838b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/734ff530e3aa44d6bf1f28e796d4838b 2024-12-10T20:17:41,894 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/b47747ee5bb041099e105352ef742929 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/b47747ee5bb041099e105352ef742929 2024-12-10T20:17:41,896 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a76e3eb2b3d0462bb2201b728974eecd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/a76e3eb2b3d0462bb2201b728974eecd 2024-12-10T20:17:41,896 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f0d4a19686fd4e16ac6ef0b77c642a47 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/f0d4a19686fd4e16ac6ef0b77c642a47 2024-12-10T20:17:41,896 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/3af713f57e194b7abb9ec33b7dc77196 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/3af713f57e194b7abb9ec33b7dc77196 2024-12-10T20:17:41,896 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/0592d52268a647c587ad42c2f4b9c52e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/0592d52268a647c587ad42c2f4b9c52e 2024-12-10T20:17:41,896 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c89ed8c4b197456bb4a7304259fea265 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c89ed8c4b197456bb4a7304259fea265 2024-12-10T20:17:41,896 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c0e47772b6cf4e1eb82619f1fb37419b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/c0e47772b6cf4e1eb82619f1fb37419b 2024-12-10T20:17:41,900 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/91f39644237444a190faf6aaf6b91d76 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/91f39644237444a190faf6aaf6b91d76 2024-12-10T20:17:41,900 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/319552ed2d7f44b5ad27a171af3dfebf to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/319552ed2d7f44b5ad27a171af3dfebf 2024-12-10T20:17:41,901 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/ed1736c324c14dea8de7d589521b7b8f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/ed1736c324c14dea8de7d589521b7b8f 2024-12-10T20:17:41,901 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/844cca8b84e6473c8d53b795d4d70496 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/844cca8b84e6473c8d53b795d4d70496 2024-12-10T20:17:41,901 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e8a781ef86b24366990bfb8c3ff63492 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/e8a781ef86b24366990bfb8c3ff63492 2024-12-10T20:17:41,901 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/4c084e1c49654eb59e9c1da35b590e09 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/4c084e1c49654eb59e9c1da35b590e09 2024-12-10T20:17:41,901 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/9f27648f484d49a7a42d3ac923f09bd3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/9f27648f484d49a7a42d3ac923f09bd3 2024-12-10T20:17:41,901 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/32391af8d1e24792a90d8f4d55099b8a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/32391af8d1e24792a90d8f4d55099b8a 2024-12-10T20:17:41,902 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/bb15d7b8780b4ebebd5a82027a7d43b6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/bb15d7b8780b4ebebd5a82027a7d43b6 2024-12-10T20:17:41,902 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fd6d152940924034aebe621b073fca20 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/fd6d152940924034aebe621b073fca20 2024-12-10T20:17:41,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/33079cfccbb04c43be18165709872258, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6c6e403cd23f4c9aadf267290e15588b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6414f99180bc4e36a245b6d9febacd2d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/1b8649c192fb47d7873b571d28fb34f3, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d36fcd58f76a45a480b2c395fdf012db, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9d842a7067a64ae2ad2908a0a1dd3bbd, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/e19b779735f549988bbe386617c74762, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ed3372386d744fa988e8cc41ef77c2df, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/cc155d7c7f224c7d92ab38452ec66501, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/15c20b302bb34d08b9ebf9f046742d6d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dc0ef5baafcf400a9cac17c472379da6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/12ff2f3aec9542cda6af2863f7645d1f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/490b030edc6d4d709b4822854300d101, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b3f0f6263edd4c40bea663a1e42584a4, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac1bb4f6878e431a85289afe456976f6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0eeb45076bfe4e0284618e52f5dccb10, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3aa5a139e63b4bc5a52938b8303be194, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0e485ca5a43643c8b6dc65bd1da25f1f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dce532ee4a24495484f54dc4f71232c6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b2412147010d4c4e9d9df4f9b89418ac, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f73ff610ca664144a71975e2b112a00a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/52760339b93e498faa7f5e0b338b770c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4d8555c751444313b13cd0e84ee9846e, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f4e6c070ef5946cfa2844e2c03a5f7d6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/514287192e33483280c383882431927a, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/027e8755bfbb4f58bc8e99e06fd85c68, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4bd5eb5351ad45418b63c31ebc38c7de, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9c5eea3b87d542b8bff8e0212788c1c6, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ddb686ed1d074a97b046c4b4bb302044] to archive 2024-12-10T20:17:41,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:17:41,908 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6c6e403cd23f4c9aadf267290e15588b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6c6e403cd23f4c9aadf267290e15588b 2024-12-10T20:17:41,908 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6414f99180bc4e36a245b6d9febacd2d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/6414f99180bc4e36a245b6d9febacd2d 2024-12-10T20:17:41,908 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d36fcd58f76a45a480b2c395fdf012db to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d36fcd58f76a45a480b2c395fdf012db 2024-12-10T20:17:41,908 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/1b8649c192fb47d7873b571d28fb34f3 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/1b8649c192fb47d7873b571d28fb34f3 2024-12-10T20:17:41,910 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9d842a7067a64ae2ad2908a0a1dd3bbd to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9d842a7067a64ae2ad2908a0a1dd3bbd 2024-12-10T20:17:41,911 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/33079cfccbb04c43be18165709872258 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/33079cfccbb04c43be18165709872258 2024-12-10T20:17:41,911 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/cc155d7c7f224c7d92ab38452ec66501 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/cc155d7c7f224c7d92ab38452ec66501 2024-12-10T20:17:41,912 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ed3372386d744fa988e8cc41ef77c2df to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ed3372386d744fa988e8cc41ef77c2df 2024-12-10T20:17:41,912 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dc0ef5baafcf400a9cac17c472379da6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dc0ef5baafcf400a9cac17c472379da6 2024-12-10T20:17:41,913 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b3f0f6263edd4c40bea663a1e42584a4 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b3f0f6263edd4c40bea663a1e42584a4 2024-12-10T20:17:41,913 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/490b030edc6d4d709b4822854300d101 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/490b030edc6d4d709b4822854300d101 2024-12-10T20:17:41,913 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/e19b779735f549988bbe386617c74762 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/e19b779735f549988bbe386617c74762 2024-12-10T20:17:41,913 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac1bb4f6878e431a85289afe456976f6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac1bb4f6878e431a85289afe456976f6 2024-12-10T20:17:41,914 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/15c20b302bb34d08b9ebf9f046742d6d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/15c20b302bb34d08b9ebf9f046742d6d 2024-12-10T20:17:41,914 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0eeb45076bfe4e0284618e52f5dccb10 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0eeb45076bfe4e0284618e52f5dccb10 2024-12-10T20:17:41,915 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/12ff2f3aec9542cda6af2863f7645d1f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/12ff2f3aec9542cda6af2863f7645d1f 2024-12-10T20:17:41,915 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3aa5a139e63b4bc5a52938b8303be194 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3aa5a139e63b4bc5a52938b8303be194 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f4e6c070ef5946cfa2844e2c03a5f7d6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f4e6c070ef5946cfa2844e2c03a5f7d6 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4d8555c751444313b13cd0e84ee9846e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4d8555c751444313b13cd0e84ee9846e 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f73ff610ca664144a71975e2b112a00a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/f73ff610ca664144a71975e2b112a00a 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/52760339b93e498faa7f5e0b338b770c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/52760339b93e498faa7f5e0b338b770c 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0e485ca5a43643c8b6dc65bd1da25f1f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/0e485ca5a43643c8b6dc65bd1da25f1f 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b2412147010d4c4e9d9df4f9b89418ac to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/b2412147010d4c4e9d9df4f9b89418ac 2024-12-10T20:17:41,918 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dce532ee4a24495484f54dc4f71232c6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/dce532ee4a24495484f54dc4f71232c6 2024-12-10T20:17:41,919 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/514287192e33483280c383882431927a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/514287192e33483280c383882431927a 2024-12-10T20:17:41,919 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4bd5eb5351ad45418b63c31ebc38c7de to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/4bd5eb5351ad45418b63c31ebc38c7de 2024-12-10T20:17:41,921 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9c5eea3b87d542b8bff8e0212788c1c6 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/9c5eea3b87d542b8bff8e0212788c1c6 2024-12-10T20:17:41,921 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/027e8755bfbb4f58bc8e99e06fd85c68 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/027e8755bfbb4f58bc8e99e06fd85c68 2024-12-10T20:17:41,921 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ddb686ed1d074a97b046c4b4bb302044 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ddb686ed1d074a97b046c4b4bb302044 2024-12-10T20:17:41,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/7c688f388eed4fe5998acf1587e7d643, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/1340701cfb01485d95f0a43b2da7e6ab, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/a7be9eedfdd649a782b0374d13c6d13d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/e75efeea180d46f6a954503eef1da939, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8b019d1d4bf74f53825065bbdf87549f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5e05df6f9b264a968b5e49e69e84198d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5d0451da506d41cca03702092acef5f9, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/cad61b015cc248b0be9639543060c6d0, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/44af10b247874082b9537676ad790905, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/88ca1a9b068740a29158c21f06c5aca8, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/78160d9e33f04c55bf985e38f5ba9c0f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/4db51b4d303542a3b7bbea1a684c9afe, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/53c1c798ab994a3296313b98995e7469, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/898f542f54f94a8592c0c51b40c2dd94, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3621ac27e6d34d0cb3eb1c065f651431, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8f5fa0095c5b466996446f9c71652204, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/19f105a6502d4b8093e98d9239ce5e07, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/63975a00a4a745a0aa09f0aa4d508dec, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca2459d2bc6b49f39cf8d13984c65816, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9de0673ee02c42e2853e96ff24a618c5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/03da6c4af9f140379f9d906f2b539d3c, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8819b93e7491486da8a0a462f1c8b6ed, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/0c2146681f934a92b2dd7ce12d62ab66, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/503b6f4f4a1d41848b89af2f7d211e9d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca5802ebb212406fa80a80689c071e86, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9deef7fe43094e7b989f01594b4fb7e5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/07635484a99b4f3fbc66d5aa3689b11b, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8549226d38d74180ada5bace4a38e20f, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/43eefebf1b9e471c96fabd1ce717ee5d] to archive 2024-12-10T20:17:41,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:17:41,931 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/a7be9eedfdd649a782b0374d13c6d13d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/a7be9eedfdd649a782b0374d13c6d13d 2024-12-10T20:17:41,931 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5d0451da506d41cca03702092acef5f9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5d0451da506d41cca03702092acef5f9 2024-12-10T20:17:41,932 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8b019d1d4bf74f53825065bbdf87549f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8b019d1d4bf74f53825065bbdf87549f 2024-12-10T20:17:41,932 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5e05df6f9b264a968b5e49e69e84198d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/5e05df6f9b264a968b5e49e69e84198d 2024-12-10T20:17:41,932 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/7c688f388eed4fe5998acf1587e7d643 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/7c688f388eed4fe5998acf1587e7d643 2024-12-10T20:17:41,933 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/e75efeea180d46f6a954503eef1da939 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/e75efeea180d46f6a954503eef1da939 2024-12-10T20:17:41,934 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/1340701cfb01485d95f0a43b2da7e6ab to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/1340701cfb01485d95f0a43b2da7e6ab 2024-12-10T20:17:41,934 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/cad61b015cc248b0be9639543060c6d0 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/cad61b015cc248b0be9639543060c6d0 2024-12-10T20:17:41,935 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/78160d9e33f04c55bf985e38f5ba9c0f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/78160d9e33f04c55bf985e38f5ba9c0f 2024-12-10T20:17:41,935 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/88ca1a9b068740a29158c21f06c5aca8 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/88ca1a9b068740a29158c21f06c5aca8 2024-12-10T20:17:41,938 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/898f542f54f94a8592c0c51b40c2dd94 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/898f542f54f94a8592c0c51b40c2dd94 2024-12-10T20:17:41,938 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/4db51b4d303542a3b7bbea1a684c9afe to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/4db51b4d303542a3b7bbea1a684c9afe 2024-12-10T20:17:41,938 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/53c1c798ab994a3296313b98995e7469 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/53c1c798ab994a3296313b98995e7469 2024-12-10T20:17:41,938 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3621ac27e6d34d0cb3eb1c065f651431 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3621ac27e6d34d0cb3eb1c065f651431 2024-12-10T20:17:41,938 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/44af10b247874082b9537676ad790905 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/44af10b247874082b9537676ad790905 2024-12-10T20:17:41,939 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8f5fa0095c5b466996446f9c71652204 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8f5fa0095c5b466996446f9c71652204 2024-12-10T20:17:41,939 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/19f105a6502d4b8093e98d9239ce5e07 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/19f105a6502d4b8093e98d9239ce5e07 2024-12-10T20:17:41,939 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca2459d2bc6b49f39cf8d13984c65816 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca2459d2bc6b49f39cf8d13984c65816 2024-12-10T20:17:41,941 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/63975a00a4a745a0aa09f0aa4d508dec to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/63975a00a4a745a0aa09f0aa4d508dec 2024-12-10T20:17:41,940 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8819b93e7491486da8a0a462f1c8b6ed to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8819b93e7491486da8a0a462f1c8b6ed 2024-12-10T20:17:41,941 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/0c2146681f934a92b2dd7ce12d62ab66 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/0c2146681f934a92b2dd7ce12d62ab66 2024-12-10T20:17:41,942 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/03da6c4af9f140379f9d906f2b539d3c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/03da6c4af9f140379f9d906f2b539d3c 2024-12-10T20:17:41,942 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/503b6f4f4a1d41848b89af2f7d211e9d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/503b6f4f4a1d41848b89af2f7d211e9d 2024-12-10T20:17:41,943 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9de0673ee02c42e2853e96ff24a618c5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9de0673ee02c42e2853e96ff24a618c5 2024-12-10T20:17:41,943 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca5802ebb212406fa80a80689c071e86 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/ca5802ebb212406fa80a80689c071e86 2024-12-10T20:17:41,943 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9deef7fe43094e7b989f01594b4fb7e5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/9deef7fe43094e7b989f01594b4fb7e5 2024-12-10T20:17:41,943 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8549226d38d74180ada5bace4a38e20f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/8549226d38d74180ada5bace4a38e20f 2024-12-10T20:17:41,943 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/07635484a99b4f3fbc66d5aa3689b11b to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/07635484a99b4f3fbc66d5aa3689b11b 2024-12-10T20:17:41,943 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/43eefebf1b9e471c96fabd1ce717ee5d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/43eefebf1b9e471c96fabd1ce717ee5d 2024-12-10T20:17:41,961 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/recovered.edits/439.seqid, newMaxSeqId=439, maxSeqId=1 2024-12-10T20:17:41,962 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. 2024-12-10T20:17:41,962 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 3412e855b2a7dd08270299896873abf4: Waiting for close lock at 1733861860600Running coprocessor pre-close hooks at 1733861860600Disabling compacts and flushes for region at 1733861860600Disabling writes for close at 1733861860600Obtaining lock to block concurrent updates at 1733861860600Preparing flush snapshotting stores in 3412e855b2a7dd08270299896873abf4 at 1733861860600Finished memstore snapshotting TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733861860600Flushing stores of TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4. at 1733861860601 (+1 ms)Flushing 3412e855b2a7dd08270299896873abf4/A: creating writer at 1733861860601Flushing 3412e855b2a7dd08270299896873abf4/A: appending metadata at 1733861860604 (+3 ms)Flushing 3412e855b2a7dd08270299896873abf4/A: closing flushed file at 1733861860604Flushing 3412e855b2a7dd08270299896873abf4/B: creating writer at 1733861861017 (+413 ms)Flushing 3412e855b2a7dd08270299896873abf4/B: appending metadata at 1733861861020 (+3 ms)Flushing 3412e855b2a7dd08270299896873abf4/B: closing flushed file at 1733861861020Flushing 3412e855b2a7dd08270299896873abf4/C: creating writer at 1733861861429 (+409 ms)Flushing 3412e855b2a7dd08270299896873abf4/C: appending metadata at 1733861861437 (+8 ms)Flushing 3412e855b2a7dd08270299896873abf4/C: closing flushed file at 1733861861437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c7dad34: reopening flushed file at 1733861861851 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a0c1f93: reopening flushed file at 1733861861858 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11fa9f98: reopening flushed file at 1733861861866 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3412e855b2a7dd08270299896873abf4 in 1272ms, sequenceid=436, compaction requested=true at 1733861861872 (+6 ms)Writing region close event to WAL at 1733861861949 (+77 ms)Running coprocessor post-close hooks at 1733861861962 (+13 ms)Closed at 1733861861962 2024-12-10T20:17:41,966 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=3412e855b2a7dd08270299896873abf4, regionState=CLOSED 2024-12-10T20:17:41,966 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:41,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:17:41,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-10T20:17:41,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 3412e855b2a7dd08270299896873abf4, server=03a2ae042c95,38133,1733861689839 in 1.5260 sec 2024-12-10T20:17:41,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-10T20:17:41,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3412e855b2a7dd08270299896873abf4, UNASSIGN in 1.5310 sec 2024-12-10T20:17:41,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-10T20:17:41,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5360 sec 2024-12-10T20:17:41,986 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861861986"}]},"ts":"1733861861986"} 2024-12-10T20:17:41,988 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T20:17:41,988 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T20:17:41,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5500 sec 2024-12-10T20:17:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T20:17:42,588 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:42,589 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T20:17:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,592 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,593 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T20:17:42,596 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:42,599 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/recovered.edits] 2024-12-10T20:17:42,603 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/d754ff86006f443a93e9832f4af2a6ad to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/d754ff86006f443a93e9832f4af2a6ad 2024-12-10T20:17:42,603 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/dff612f37c7745429434794a22fc6311 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/dff612f37c7745429434794a22fc6311 2024-12-10T20:17:42,603 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/2a94371e42d646cf982e0abdc856d96e to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/A/2a94371e42d646cf982e0abdc856d96e 2024-12-10T20:17:42,604 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3cce43ffce3b45b4b4cd4c64cae38b8f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/3cce43ffce3b45b4b4cd4c64cae38b8f 2024-12-10T20:17:42,605 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d3ded8c38ad844edaaae28c289fcd465 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/d3ded8c38ad844edaaae28c289fcd465 2024-12-10T20:17:42,605 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac8c973773434d17942b4dbcb46d6f44 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/B/ac8c973773434d17942b4dbcb46d6f44 2024-12-10T20:17:42,607 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/260ec70cea4644b281ad217bb09b35df to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/260ec70cea4644b281ad217bb09b35df 2024-12-10T20:17:42,607 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3aa710fda8a849d693accb897822cc1f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/3aa710fda8a849d693accb897822cc1f 2024-12-10T20:17:42,607 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/fe2dda13febb4308818b7b591c326ac7 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/C/fe2dda13febb4308818b7b591c326ac7 2024-12-10T20:17:42,609 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/recovered.edits/439.seqid to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4/recovered.edits/439.seqid 2024-12-10T20:17:42,609 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/3412e855b2a7dd08270299896873abf4 2024-12-10T20:17:42,609 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T20:17:42,611 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,612 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T20:17:42,614 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T20:17:42,615 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,615 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T20:17:42,615 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733861862615"}]},"ts":"9223372036854775807"} 2024-12-10T20:17:42,616 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T20:17:42,616 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3412e855b2a7dd08270299896873abf4, NAME => 'TestAcidGuarantees,,1733861829304.3412e855b2a7dd08270299896873abf4.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T20:17:42,616 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T20:17:42,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733861862616"}]},"ts":"9223372036854775807"} 2024-12-10T20:17:42,618 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T20:17:42,618 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-12-10T20:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T20:17:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T20:17:42,697 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:42,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 253), OpenFileDescriptor=441 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=859 (was 839) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1273 (was 3123) 2024-12-10T20:17:42,722 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=859, ProcessCount=11, AvailableMemoryMB=1272 2024-12-10T20:17:42,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:17:42,723 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:42,725 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:17:42,725 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:42,725 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-12-10T20:17:42,725 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T20:17:42,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742368_1544 (size=1061) 2024-12-10T20:17:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T20:17:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T20:17:43,131 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e 2024-12-10T20:17:43,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742369_1545 (size=53) 2024-12-10T20:17:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T20:17:43,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:17:43,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4247c586a6941b4d0687e2fa49098b0a, disabling compactions & flushes 2024-12-10T20:17:43,535 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,536 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. after waiting 0 ms 2024-12-10T20:17:43,536 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,536 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,536 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4247c586a6941b4d0687e2fa49098b0a: Waiting for close lock at 1733861863535Disabling compacts and flushes for region at 1733861863535Disabling writes for close at 1733861863536 (+1 ms)Writing region close event to WAL at 1733861863536Closed at 1733861863536 2024-12-10T20:17:43,537 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:17:43,537 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733861863537"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733861863537"}]},"ts":"1733861863537"} 2024-12-10T20:17:43,539 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:17:43,540 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:17:43,540 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861863540"}]},"ts":"1733861863540"} 2024-12-10T20:17:43,541 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T20:17:43,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, ASSIGN}] 2024-12-10T20:17:43,543 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, ASSIGN 2024-12-10T20:17:43,543 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, ASSIGN; state=OFFLINE, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=false 2024-12-10T20:17:43,694 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:43,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, ASSIGN because future has completed 2024-12-10T20:17:43,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:17:43,854 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,855 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:17:43,855 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,855 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:17:43,855 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,855 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,857 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,860 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:43,860 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4247c586a6941b4d0687e2fa49098b0a columnFamilyName A 2024-12-10T20:17:43,860 DEBUG [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:43,861 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(327): Store=4247c586a6941b4d0687e2fa49098b0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:43,861 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,863 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:43,863 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4247c586a6941b4d0687e2fa49098b0a columnFamilyName B 2024-12-10T20:17:43,863 DEBUG [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:43,865 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(327): Store=4247c586a6941b4d0687e2fa49098b0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:43,865 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,866 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:43,867 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4247c586a6941b4d0687e2fa49098b0a columnFamilyName C 2024-12-10T20:17:43,867 DEBUG [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T20:17:43,868 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(327): Store=4247c586a6941b4d0687e2fa49098b0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:43,868 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,868 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,869 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,869 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,871 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,871 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,872 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:17:43,874 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,876 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:17:43,876 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 4247c586a6941b4d0687e2fa49098b0a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60088577, jitterRate=-0.10461042821407318}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:17:43,876 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:43,877 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 4247c586a6941b4d0687e2fa49098b0a: Running coprocessor pre-open hook at 1733861863855Writing region info on filesystem at 1733861863855Initializing all the Stores at 1733861863857 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861863857Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861863857Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861863857Cleaning up temporary data from old regions at 1733861863871 (+14 ms)Running coprocessor post-open hooks at 1733861863876 (+5 ms)Region opened successfully at 1733861863877 (+1 ms) 2024-12-10T20:17:43,878 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., pid=138, masterSystemTime=1733861863850 2024-12-10T20:17:43,879 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,879 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:43,880 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:43,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:17:43,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-10T20:17:43,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 in 185 msec 2024-12-10T20:17:43,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-10T20:17:43,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, ASSIGN in 341 msec 2024-12-10T20:17:43,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:17:43,886 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861863886"}]},"ts":"1733861863886"} 2024-12-10T20:17:43,888 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T20:17:43,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:17:43,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-12-10T20:17:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T20:17:44,878 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:44,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c11d8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:44,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:44,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:44,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ab5178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:44,885 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa16a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:44,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:44,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:44,890 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:17:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,45643,1733861688870 2024-12-10T20:17:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12217bce 2024-12-10T20:17:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:17:44,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:17:44,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T20:17:44,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:17:47,483 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:17:47,801 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:17:50,279 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_417660294_22 at /127.0.0.1:45052 [Receiving block BP-1506670894-172.17.0.3-1733861683702:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 5384ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2/, blockId=1073741830, seqno=1750 2024-12-10T20:17:50,281 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5385 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:17:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T20:17:50,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742370_1546 (size=1097) 2024-12-10T20:17:50,301 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T20:17:50,301 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T20:17:50,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:17:50,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, REOPEN/MOVE}] 2024-12-10T20:17:50,306 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, REOPEN/MOVE 2024-12-10T20:17:50,307 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, REOPEN/MOVE because future has completed 2024-12-10T20:17:50,308 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:17:50,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:17:50,460 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,461 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:17:50,461 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 4247c586a6941b4d0687e2fa49098b0a, disabling compactions & flushes 2024-12-10T20:17:50,461 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,461 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,461 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. after waiting 0 ms 2024-12-10T20:17:50,461 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,464 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T20:17:50,465 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,465 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 4247c586a6941b4d0687e2fa49098b0a: Waiting for close lock at 1733861870461Running coprocessor pre-close hooks at 1733861870461Disabling compacts and flushes for region at 1733861870461Disabling writes for close at 1733861870461Writing region close event to WAL at 1733861870461Running coprocessor post-close hooks at 1733861870465 (+4 ms)Closed at 1733861870465 2024-12-10T20:17:50,465 WARN [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 4247c586a6941b4d0687e2fa49098b0a to self. 2024-12-10T20:17:50,467 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=CLOSED 2024-12-10T20:17:50,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:17:50,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-10T20:17:50,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 in 162 msec 2024-12-10T20:17:50,473 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, REOPEN/MOVE; state=CLOSED, location=03a2ae042c95,38133,1733861689839; forceNewPlan=false, retain=true 2024-12-10T20:17:50,624 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=OPENING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, REOPEN/MOVE because future has completed 2024-12-10T20:17:50,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:17:50,781 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,781 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:17:50,781 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,781 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:17:50,781 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,781 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,783 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,783 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:50,783 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4247c586a6941b4d0687e2fa49098b0a columnFamilyName A 2024-12-10T20:17:50,784 DEBUG [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:50,785 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(327): Store=4247c586a6941b4d0687e2fa49098b0a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:50,785 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,785 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:50,786 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4247c586a6941b4d0687e2fa49098b0a columnFamilyName B 2024-12-10T20:17:50,786 DEBUG [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:50,786 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(327): Store=4247c586a6941b4d0687e2fa49098b0a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:50,786 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,786 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T20:17:50,787 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4247c586a6941b4d0687e2fa49098b0a columnFamilyName C 2024-12-10T20:17:50,787 DEBUG [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:50,787 INFO [StoreOpener-4247c586a6941b4d0687e2fa49098b0a-1 {}] regionserver.HStore(327): Store=4247c586a6941b4d0687e2fa49098b0a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:17:50,787 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,787 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,788 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,788 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,789 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,789 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,790 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:17:50,791 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,791 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 4247c586a6941b4d0687e2fa49098b0a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60336017, jitterRate=-0.10092328488826752}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:17:50,791 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,792 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 4247c586a6941b4d0687e2fa49098b0a: Running coprocessor pre-open hook at 1733861870782Writing region info on filesystem at 1733861870782Initializing all the Stores at 1733861870782Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861870782Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861870782Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733861870782Cleaning up temporary data from old regions at 1733861870789 (+7 ms)Running coprocessor post-open hooks at 1733861870791 (+2 ms)Region opened successfully at 1733861870792 (+1 ms) 2024-12-10T20:17:50,793 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., pid=143, masterSystemTime=1733861870778 2024-12-10T20:17:50,794 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,794 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:50,795 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=OPEN, openSeqNum=5, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:17:50,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-12-10T20:17:50,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 in 171 msec 2024-12-10T20:17:50,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-10T20:17:50,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, REOPEN/MOVE in 493 msec 2024-12-10T20:17:50,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-10T20:17:50,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-12-10T20:17:50,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 5.9110 sec 2024-12-10T20:17:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T20:17:50,810 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:17:50,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24edf963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4ace48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4358fa47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966c360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ad7841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,816 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd9136a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6379f6de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a68fa89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a9c21e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337ab365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,820 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57258284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,821 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38018ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1be13a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,823 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0ccef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@741cdfbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9aea37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,825 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5ccf99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781f2a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1903d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,827 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df1119a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,828 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31eacf87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e153e49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,830 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c547b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,832 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab01d50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158baa16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe94191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e49dfd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160e927f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,45643,-1 for getting cluster id 2024-12-10T20:17:50,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:17:50,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '034640c3-bcac-4843-ae54-831f788f9a71' 2024-12-10T20:17:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:17:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "034640c3-bcac-4843-ae54-831f788f9a71" 2024-12-10T20:17:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9d3bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,45643,-1] 2024-12-10T20:17:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:17:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:17:50,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:17:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@474129e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:17:50,842 DEBUG [Thread-2459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,842 DEBUG [Thread-2448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,842 DEBUG [Thread-2463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,842 DEBUG [Thread-2461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,842 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,842 DEBUG [Thread-2454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,843 DEBUG [Thread-2452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [Thread-2459 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,844 DEBUG [Thread-2465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,844 DEBUG [Thread-2450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,844 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:17:50,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,843 DEBUG [Thread-2459 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,845 DEBUG [Thread-2465 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T20:17:50,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,845 DEBUG [Thread-2465 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,38133,1733861689839, seqNum=-1] 2024-12-10T20:17:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-10T20:17:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:17:50,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:50,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,849 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T20:17:50,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:17:50,850 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T20:17:50,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T20:17:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5] 2024-12-10T20:17:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:50,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4247c586a6941b4d0687e2fa49098b0a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T20:17:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=A 2024-12-10T20:17:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=B 2024-12-10T20:17:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=C 2024-12-10T20:17:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:17:50,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100dd0f6904f144e928e75193ea2bef751_4247c586a6941b4d0687e2fa49098b0a is 50, key is test_row_0/A:col10/1733861870856/Put/seqid=0 2024-12-10T20:17:50,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742371_1547 (size=12154) 2024-12-10T20:17:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861930872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:50,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1733861870869, max timestamp=1733861870881Num uniques 859; 2024-12-10T20:17:50,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733861870868, max timestamp=1733861870881Num uniques 859; 2024-12-10T20:17:50,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7964, suffix heap size=117064,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=174 2024-12-10T20:17:50,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:17:50,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733861870868, max timestamp=1733861870881Num uniques 859; 2024-12-10T20:17:50,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T20:17:50,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861930912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861930913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861930916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861930917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:51,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861931007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861931027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861931028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861931028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861931028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:51,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861931217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861931236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861931236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861931236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861931236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,280 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:17:51,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100dd0f6904f144e928e75193ea2bef751_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100dd0f6904f144e928e75193ea2bef751_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:51,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/d95eab1aa583474c8f57e4dd537a4988, store: [table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:17:51,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/d95eab1aa583474c8f57e4dd537a4988 is 175, key is test_row_0/A:col10/1733861870856/Put/seqid=0 2024-12-10T20:17:51,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742372_1548 (size=30955) 2024-12-10T20:17:51,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/d95eab1aa583474c8f57e4dd537a4988 2024-12-10T20:17:51,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/0c1da7c09fc54324b187954beabe365d is 50, key is test_row_0/B:col10/1733861870856/Put/seqid=0 2024-12-10T20:17:51,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742373_1549 (size=12001) 2024-12-10T20:17:51,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861931527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861931546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861931547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861931547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861931547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:51,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/0c1da7c09fc54324b187954beabe365d 2024-12-10T20:17:51,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/56abf6cdf2834e85b55d705d7d5367b2 is 50, key is test_row_0/C:col10/1733861870856/Put/seqid=0 2024-12-10T20:17:51,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742374_1550 (size=12001) 2024-12-10T20:17:51,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:51,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:52,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861932036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861932057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:52,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861932057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861932057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861932057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:52,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:52,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:52,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/56abf6cdf2834e85b55d705d7d5367b2 2024-12-10T20:17:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/d95eab1aa583474c8f57e4dd537a4988 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988 2024-12-10T20:17:52,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988, entries=150, sequenceid=15, filesize=30.2 K 2024-12-10T20:17:52,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/0c1da7c09fc54324b187954beabe365d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/0c1da7c09fc54324b187954beabe365d 2024-12-10T20:17:52,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/0c1da7c09fc54324b187954beabe365d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-10T20:17:52,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/56abf6cdf2834e85b55d705d7d5367b2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/56abf6cdf2834e85b55d705d7d5367b2 2024-12-10T20:17:52,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/56abf6cdf2834e85b55d705d7d5367b2, entries=150, sequenceid=15, filesize=11.7 K 2024-12-10T20:17:52,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:17:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:17:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:17:52,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:17:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:55,968 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:17:57,694 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-936230941_22 at /127.0.0.1:45064 [Receiving block BP-1506670894-172.17.0.3-1733861683702:blk_1073741833_1009] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 5546ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data1/, blockId=1073741833, seqno=2168 2024-12-10T20:17:57,695 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e-prefix:03a2ae042c95,38133,1733861689839 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5546 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:17:57,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=130.38 KB/133510 for 4247c586a6941b4d0687e2fa49098b0a in 6839ms, sequenceid=15, compaction requested=false 2024-12-10T20:17:57,696 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e-prefix:03a2ae042c95,38133,1733861689839 {}] wal.AbstractFSWAL(1368): Slow sync cost: 4634 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:17:57,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:17:57,696 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e-prefix:03a2ae042c95,38133,1733861689839 {}] wal.AbstractFSWAL(1368): Slow sync cost: 4627 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:17:57,696 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e-prefix:03a2ae042c95,38133,1733861689839 {}] wal.AbstractFSWAL(1368): Slow sync cost: 4625 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:17:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:17:57,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4247c586a6941b4d0687e2fa49098b0a 3/3 column families, dataSize=150.51 KB heapSize=427.45 KB 2024-12-10T20:17:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733861870910, max timestamp=1733861873070Num uniques 1118; 2024-12-10T20:17:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733861870910, max timestamp=1733861873070Num uniques 1118; 2024-12-10T20:17:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-10T20:17:57,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/03a2ae042c95:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-10T20:17:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=A 2024-12-10T20:17:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=B 2024-12-10T20:17:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T20:17:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=C 2024-12-10T20:17:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T20:17:57,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861937713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861937718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861937718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861937719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861937719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861937826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861937827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861937827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861937827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861937827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861938036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861938037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861938037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861938037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861938037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861938347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861938347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861938347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861938347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861938347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861938857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861938857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861938857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861938857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:58,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861938858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:17:59,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861939866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:59,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861939866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:59,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861939866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:59,869 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:59,869 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:59,869 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:59,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861939877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:59,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:17:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861939877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:17:59,880 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:17:59,880 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:17:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:17:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861941877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:01,879 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:01,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861941886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861941886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:01,889 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:01,889 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861941897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:01,899 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:01,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861941907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:01,908 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:05,002 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_417660294_22 at /127.0.0.1:45052 [Receiving block BP-1506670894-172.17.0.3-1733861683702:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 12777ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2/, blockId=1073741830, seqno=1845 2024-12-10T20:18:05,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK], data length=549, duration=12778ms, unfinishedReplicas=0, lastAckTimestamp=-1, monitor name: defaultMonitorName 2024-12-10T20:18:05,006 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1368): Slow sync cost: 12781 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:18:05,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121041fde26bc9244777aa4048e266f16f0e_4247c586a6941b4d0687e2fa49098b0a is 50, key is test_row_0/A:col10/1733861877698/Put/seqid=0 2024-12-10T20:18:05,009 WARN [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=12781 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:18:05,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742375_1551 (size=14594) 2024-12-10T20:18:05,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:18:05,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121041fde26bc9244777aa4048e266f16f0e_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121041fde26bc9244777aa4048e266f16f0e_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:05,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/07875ea8c1854419a6bb8f3f7eebb4d5, store: [table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:05,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/07875ea8c1854419a6bb8f3f7eebb4d5 is 175, key is test_row_0/A:col10/1733861877698/Put/seqid=0 2024-12-10T20:18:05,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742376_1552 (size=39549) 2024-12-10T20:18:05,110 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 03a2ae042c95%2C45643%2C1733861688870:(num 1733861690938) roll requested 2024-12-10T20:18:05,128 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861885110, exclude list is [], retry=0 2024-12-10T20:18:05,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK] 2024-12-10T20:18:05,133 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861690938 with entries=1882, filesize=819.14 KB; new WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861885110 2024-12-10T20:18:05,134 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42885:42885)] 2024-12-10T20:18:05,134 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861690938 is not closed yet, will try archiving it next time 2024-12-10T20:18:05,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741830_1006 (size=838810) 2024-12-10T20:18:05,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/07875ea8c1854419a6bb8f3f7eebb4d5 2024-12-10T20:18:05,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/c181830171194eeabec2bace3a170157 is 50, key is test_row_0/B:col10/1733861877698/Put/seqid=0 2024-12-10T20:18:05,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742378_1554 (size=12001) 2024-12-10T20:18:05,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:05,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:05,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/c181830171194eeabec2bace3a170157 2024-12-10T20:18:05,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/f602202bac87428fa409eae9ca7c156d is 50, key is test_row_0/C:col10/1733861877698/Put/seqid=0 2024-12-10T20:18:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742379_1555 (size=12001) 2024-12-10T20:18:05,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.87 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/f602202bac87428fa409eae9ca7c156d 2024-12-10T20:18:05,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/07875ea8c1854419a6bb8f3f7eebb4d5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5 2024-12-10T20:18:05,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5, entries=200, sequenceid=54, filesize=38.6 K 2024-12-10T20:18:05,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/c181830171194eeabec2bace3a170157 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/c181830171194eeabec2bace3a170157 2024-12-10T20:18:05,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/c181830171194eeabec2bace3a170157, entries=150, sequenceid=54, filesize=11.7 K 2024-12-10T20:18:05,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/f602202bac87428fa409eae9ca7c156d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/f602202bac87428fa409eae9ca7c156d 2024-12-10T20:18:05,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/f602202bac87428fa409eae9ca7c156d, entries=150, sequenceid=54, filesize=11.7 K 2024-12-10T20:18:05,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~81.18 KB/83130, heapSize ~220.51 KB/225800, currentSize=100.63 KB/103050 for 4247c586a6941b4d0687e2fa49098b0a in 8193ms, sequenceid=54, compaction requested=false 2024-12-10T20:18:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:05,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4247c586a6941b4d0687e2fa49098b0a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T20:18:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=A 2024-12-10T20:18:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=B 2024-12-10T20:18:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=C 2024-12-10T20:18:05,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:05,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861945918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:05,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:05,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:05,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:05,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861945921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861945921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861945926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:05,929 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:05,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fb5dacbf418641eb986d62bc5a22b8de_4247c586a6941b4d0687e2fa49098b0a is 50, key is test_row_0/A:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:05,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861945937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:05,939 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:05,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742380_1556 (size=14594) 2024-12-10T20:18:05,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:18:05,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fb5dacbf418641eb986d62bc5a22b8de_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fb5dacbf418641eb986d62bc5a22b8de_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:05,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/4baf7f9a01034525945e4ae675fa8436, store: [table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:05,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/4baf7f9a01034525945e4ae675fa8436 is 175, key is test_row_0/A:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:05,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742381_1557 (size=39549) 2024-12-10T20:18:05,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=74, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/4baf7f9a01034525945e4ae675fa8436 2024-12-10T20:18:05,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/19bcc28170764c7b8c430249e3151dc2 is 50, key is test_row_0/B:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742382_1558 (size=12001) 2024-12-10T20:18:05,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/19bcc28170764c7b8c430249e3151dc2 2024-12-10T20:18:05,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/87053aca3e054f5bac33c092790c3198 is 50, key is test_row_0/C:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:05,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742383_1559 (size=12001) 2024-12-10T20:18:05,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/87053aca3e054f5bac33c092790c3198 2024-12-10T20:18:06,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/4baf7f9a01034525945e4ae675fa8436 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436 2024-12-10T20:18:06,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436, entries=200, sequenceid=74, filesize=38.6 K 2024-12-10T20:18:06,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/19bcc28170764c7b8c430249e3151dc2 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/19bcc28170764c7b8c430249e3151dc2 2024-12-10T20:18:06,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/19bcc28170764c7b8c430249e3151dc2, entries=150, sequenceid=74, filesize=11.7 K 2024-12-10T20:18:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/87053aca3e054f5bac33c092790c3198 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/87053aca3e054f5bac33c092790c3198 2024-12-10T20:18:06,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/87053aca3e054f5bac33c092790c3198, entries=150, sequenceid=74, filesize=11.7 K 2024-12-10T20:18:06,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 4247c586a6941b4d0687e2fa49098b0a in 117ms, sequenceid=74, compaction requested=true 2024-12-10T20:18:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4247c586a6941b4d0687e2fa49098b0a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T20:18:06,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:18:06,017 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:18:06,017 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:18:06,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4247c586a6941b4d0687e2fa49098b0a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T20:18:06,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:18:06,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4247c586a6941b4d0687e2fa49098b0a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T20:18:06,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:18:06,017 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:18:06,017 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:18:06,018 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 4247c586a6941b4d0687e2fa49098b0a/B is initiating minor compaction (all files) 2024-12-10T20:18:06,018 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1541): 4247c586a6941b4d0687e2fa49098b0a/A is initiating minor compaction (all files) 2024-12-10T20:18:06,018 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4247c586a6941b4d0687e2fa49098b0a/B in TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,018 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4247c586a6941b4d0687e2fa49098b0a/A in TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,018 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/0c1da7c09fc54324b187954beabe365d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/c181830171194eeabec2bace3a170157, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/19bcc28170764c7b8c430249e3151dc2] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp, totalSize=35.2 K 2024-12-10T20:18:06,018 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp, totalSize=107.5 K 2024-12-10T20:18:06,018 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,018 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. files: [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436] 2024-12-10T20:18:06,019 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95eab1aa583474c8f57e4dd537a4988, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733861870854 2024-12-10T20:18:06,019 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1da7c09fc54324b187954beabe365d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733861870854 2024-12-10T20:18:06,019 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07875ea8c1854419a6bb8f3f7eebb4d5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733861870912 2024-12-10T20:18:06,019 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting c181830171194eeabec2bace3a170157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733861873068 2024-12-10T20:18:06,020 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4baf7f9a01034525945e4ae675fa8436, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733861877718 2024-12-10T20:18:06,020 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bcc28170764c7b8c430249e3151dc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733861877718 2024-12-10T20:18:06,026 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4247c586a6941b4d0687e2fa49098b0a#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:18:06,027 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/69f82218bfd64ea1b0792971d72435a9 is 50, key is test_row_0/B:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(8855): Flush requested on 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:06,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4247c586a6941b4d0687e2fa49098b0a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T20:18:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=A 2024-12-10T20:18:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=B 2024-12-10T20:18:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=C 2024-12-10T20:18:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:06,035 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:06,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742384_1560 (size=12104) 2024-12-10T20:18:06,045 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121006137d9f0f4c4450b2b533b372d0e519_4247c586a6941b4d0687e2fa49098b0a store=[table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:06,046 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121006137d9f0f4c4450b2b533b372d0e519_4247c586a6941b4d0687e2fa49098b0a, store=[table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:06,046 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121006137d9f0f4c4450b2b533b372d0e519_4247c586a6941b4d0687e2fa49098b0a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:06,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d151bcdf70974628936edab91fb3ed21_4247c586a6941b4d0687e2fa49098b0a is 50, key is test_row_0/A:col10/1733861885919/Put/seqid=0 2024-12-10T20:18:06,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861946057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861946058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861946060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742385_1561 (size=4469) 2024-12-10T20:18:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742386_1562 (size=12154) 2024-12-10T20:18:06,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861946166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861946166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861946177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861946377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861946379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861946386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,443 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/69f82218bfd64ea1b0792971d72435a9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/69f82218bfd64ea1b0792971d72435a9 2024-12-10T20:18:06,446 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4247c586a6941b4d0687e2fa49098b0a/B of 4247c586a6941b4d0687e2fa49098b0a into 69f82218bfd64ea1b0792971d72435a9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:18:06,446 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:06,446 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., storeName=4247c586a6941b4d0687e2fa49098b0a/B, priority=13, startTime=1733861886017; duration=0sec 2024-12-10T20:18:06,446 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T20:18:06,446 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4247c586a6941b4d0687e2fa49098b0a:B 2024-12-10T20:18:06,446 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T20:18:06,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T20:18:06,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1541): 4247c586a6941b4d0687e2fa49098b0a/C is initiating minor compaction (all files) 2024-12-10T20:18:06,447 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4247c586a6941b4d0687e2fa49098b0a/C in TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,447 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/56abf6cdf2834e85b55d705d7d5367b2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/f602202bac87428fa409eae9ca7c156d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/87053aca3e054f5bac33c092790c3198] into tmpdir=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp, totalSize=35.2 K 2024-12-10T20:18:06,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 56abf6cdf2834e85b55d705d7d5367b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733861870854 2024-12-10T20:18:06,447 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting f602202bac87428fa409eae9ca7c156d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733861873068 2024-12-10T20:18:06,448 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] compactions.Compactor(225): Compacting 87053aca3e054f5bac33c092790c3198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733861877718 2024-12-10T20:18:06,453 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4247c586a6941b4d0687e2fa49098b0a#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:18:06,453 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/d9ce93fdd0ad43dda1293082b376e58d is 50, key is test_row_0/C:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:06,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742387_1563 (size=12104) 2024-12-10T20:18:06,491 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4247c586a6941b4d0687e2fa49098b0a#A#compaction#481 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:18:06,491 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/a028360126b74ade9d2e749e04434ee9 is 175, key is test_row_0/A:col10/1733861885898/Put/seqid=0 2024-12-10T20:18:06,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742388_1564 (size=31058) 2024-12-10T20:18:06,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:18:06,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d151bcdf70974628936edab91fb3ed21_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d151bcdf70974628936edab91fb3ed21_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:06,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/3b531e5fae514613b34df90bacca1b11, store: [table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:06,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/3b531e5fae514613b34df90bacca1b11 is 175, key is test_row_0/A:col10/1733861885919/Put/seqid=0 2024-12-10T20:18:06,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742389_1565 (size=30955) 2024-12-10T20:18:06,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861946687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861946687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861946697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:06,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,859 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/d9ce93fdd0ad43dda1293082b376e58d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/d9ce93fdd0ad43dda1293082b376e58d 2024-12-10T20:18:06,862 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4247c586a6941b4d0687e2fa49098b0a/C of 4247c586a6941b4d0687e2fa49098b0a into d9ce93fdd0ad43dda1293082b376e58d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:18:06,862 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:06,862 INFO [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., storeName=4247c586a6941b4d0687e2fa49098b0a/C, priority=13, startTime=1733861886017; duration=0sec 2024-12-10T20:18:06,862 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:18:06,862 DEBUG [RS:0;03a2ae042c95:38133-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4247c586a6941b4d0687e2fa49098b0a:C 2024-12-10T20:18:06,897 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/a028360126b74ade9d2e749e04434ee9 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/a028360126b74ade9d2e749e04434ee9 2024-12-10T20:18:06,901 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4247c586a6941b4d0687e2fa49098b0a/A of 4247c586a6941b4d0687e2fa49098b0a into a028360126b74ade9d2e749e04434ee9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:18:06,901 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:06,901 INFO [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., storeName=4247c586a6941b4d0687e2fa49098b0a/A, priority=13, startTime=1733861886016; duration=0sec 2024-12-10T20:18:06,901 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T20:18:06,901 DEBUG [RS:0;03a2ae042c95:38133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4247c586a6941b4d0687e2fa49098b0a:A 2024-12-10T20:18:06,904 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/3b531e5fae514613b34df90bacca1b11 2024-12-10T20:18:06,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/646fcb6ea731499db54db0ecda589471 is 50, key is test_row_0/B:col10/1733861885919/Put/seqid=0 2024-12-10T20:18:06,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742390_1566 (size=12001) 2024-12-10T20:18:06,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/646fcb6ea731499db54db0ecda589471 2024-12-10T20:18:06,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/e43e5a9ecda642479707e83d5f6dd26c is 50, key is test_row_0/C:col10/1733861885919/Put/seqid=0 2024-12-10T20:18:06,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742391_1567 (size=12001) 2024-12-10T20:18:06,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:06,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:07,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. as already flushing 2024-12-10T20:18:07,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:07,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:07,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:18:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861947197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861947197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861947207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861948217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861948217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:08,219 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:08,219 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:08,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861948227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:08,229 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:18:10,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861950237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:10,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861950237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:10,239 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:10,239 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:10,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861950246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:10,249 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:10,844 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-12-10T20:18:10,844 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:10,844 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,844 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:10,844 INFO [Thread-2459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2459. 2024-12-10T20:18:10,844 DEBUG [Thread-2459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:10,844 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,844 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:10,846 INFO [Thread-2461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2461. 2024-12-10T20:18:10,846 DEBUG [Thread-2461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:10,846 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,846 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:10,846 INFO [Thread-2465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2465. 2024-12-10T20:18:10,846 DEBUG [Thread-2465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:10,846 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,846 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:10,847 INFO [Thread-2463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2463. 2024-12-10T20:18:10,847 DEBUG [Thread-2463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:10,847 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,847 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:10,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36252 deadline: 1733861954258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:14,260 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:14,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:14,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36254 deadline: 1733861954267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:14,269 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_0' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:14,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:14,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1733861954298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:14,299 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:16,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:16,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T20:18:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36318 deadline: 1733861956018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38133 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36310 deadline: 1733861956018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:16,020 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_1' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18301 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:16,020 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 03a2ae042c95,38133,1733861689839 for 'test_row_2' in 4247c586a6941b4d0687e2fa49098b0a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T20:18:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 , the old value is region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T20:18:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., hostname=03a2ae042c95,38133,1733861689839, seqNum=5 because the exception is null or not the one we care about 2024-12-10T20:18:17,801 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:18:19,019 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_417660294_22 at /127.0.0.1:52062 [Receiving block BP-1506670894-172.17.0.3-1733861683702:blk_1073742377_1553] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 11872ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data1/, blockId=1073742377, seqno=40 2024-12-10T20:18:19,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/e43e5a9ecda642479707e83d5f6dd26c 2024-12-10T20:18:19,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK], data length=750, duration=11873ms, unfinishedReplicas=0, lastAckTimestamp=-1, monitor name: defaultMonitorName 2024-12-10T20:18:19,020 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1368): Slow sync cost: 11873 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:18:19,021 WARN [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=11873 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:18:19,021 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 03a2ae042c95%2C45643%2C1733861688870:(num 1733861885110) roll requested 2024-12-10T20:18:19,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/3b531e5fae514613b34df90bacca1b11 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/3b531e5fae514613b34df90bacca1b11 2024-12-10T20:18:19,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/3b531e5fae514613b34df90bacca1b11, entries=150, sequenceid=89, filesize=30.2 K 2024-12-10T20:18:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/646fcb6ea731499db54db0ecda589471 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/646fcb6ea731499db54db0ecda589471 2024-12-10T20:18:19,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/646fcb6ea731499db54db0ecda589471, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T20:18:19,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/e43e5a9ecda642479707e83d5f6dd26c as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/e43e5a9ecda642479707e83d5f6dd26c 2024-12-10T20:18:19,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/e43e5a9ecda642479707e83d5f6dd26c, entries=150, sequenceid=89, filesize=11.7 K 2024-12-10T20:18:19,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4247c586a6941b4d0687e2fa49098b0a in 13004ms, sequenceid=89, compaction requested=false 2024-12-10T20:18:19,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:19,034 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861899021, exclude list is [], retry=0 2024-12-10T20:18:19,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK] 2024-12-10T20:18:19,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38133 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-10T20:18:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:19,172 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 4247c586a6941b4d0687e2fa49098b0a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T20:18:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=A 2024-12-10T20:18:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=B 2024-12-10T20:18:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=C 2024-12-10T20:18:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:19,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121025eb186aa1804f7da0be08059f95f49e_4247c586a6941b4d0687e2fa49098b0a is 50, key is test_row_0/A:col10/1733861886059/Put/seqid=0 2024-12-10T20:18:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742393_1569 (size=12154) 2024-12-10T20:18:19,206 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1368): Slow sync cost: 184 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:18:19,206 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861885110 with entries=41, filesize=12.54 KB; new WAL /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861899021 2024-12-10T20:18:19,206 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42885:42885)] 2024-12-10T20:18:19,206 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/WALs/03a2ae042c95,45643,1733861688870/03a2ae042c95%2C45643%2C1733861688870.1733861885110 is not closed yet, will try archiving it next time 2024-12-10T20:18:19,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742377_1553 (size=12846) 2024-12-10T20:18:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:18:19,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:18:19,587 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121025eb186aa1804f7da0be08059f95f49e_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121025eb186aa1804f7da0be08059f95f49e_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/97cc9330c68046d495bfe1a5283dbbad, store: [table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/97cc9330c68046d495bfe1a5283dbbad is 175, key is test_row_0/A:col10/1733861886059/Put/seqid=0 2024-12-10T20:18:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742394_1570 (size=30955) 2024-12-10T20:18:19,995 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/97cc9330c68046d495bfe1a5283dbbad 2024-12-10T20:18:20,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/e7eecb9efb8c42daab5c1daf5f937f48 is 50, key is test_row_0/B:col10/1733861886059/Put/seqid=0 2024-12-10T20:18:20,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742395_1571 (size=12001) 2024-12-10T20:18:20,404 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/e7eecb9efb8c42daab5c1daf5f937f48 2024-12-10T20:18:20,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/15668c072ead4138b159f2c1feef25b5 is 50, key is test_row_0/C:col10/1733861886059/Put/seqid=0 2024-12-10T20:18:20,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742396_1572 (size=12001) 2024-12-10T20:18:20,819 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/15668c072ead4138b159f2c1feef25b5 2024-12-10T20:18:20,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/97cc9330c68046d495bfe1a5283dbbad as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/97cc9330c68046d495bfe1a5283dbbad 2024-12-10T20:18:20,830 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/97cc9330c68046d495bfe1a5283dbbad, entries=150, sequenceid=109, filesize=30.2 K 2024-12-10T20:18:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/e7eecb9efb8c42daab5c1daf5f937f48 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/e7eecb9efb8c42daab5c1daf5f937f48 2024-12-10T20:18:20,835 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/e7eecb9efb8c42daab5c1daf5f937f48, entries=150, sequenceid=109, filesize=11.7 K 2024-12-10T20:18:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/15668c072ead4138b159f2c1feef25b5 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/15668c072ead4138b159f2c1feef25b5 2024-12-10T20:18:20,840 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/15668c072ead4138b159f2c1feef25b5, entries=150, sequenceid=109, filesize=11.7 K 2024-12-10T20:18:20,841 INFO [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 4247c586a6941b4d0687e2fa49098b0a in 1669ms, sequenceid=109, compaction requested=true 2024-12-10T20:18:20,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 4247c586a6941b4d0687e2fa49098b0a: 2024-12-10T20:18:20,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:20,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-10T20:18:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-10T20:18:20,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-10T20:18:20,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 29.9930 sec 2024-12-10T20:18:20,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 30.0010 sec 2024-12-10T20:18:24,288 INFO [Thread-2448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2448. 2024-12-10T20:18:24,288 INFO [Thread-2454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2454. 2024-12-10T20:18:24,288 DEBUG [Thread-2454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:24,288 DEBUG [Thread-2448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:24,288 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:24,288 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:24,288 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:24,288 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:24,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:24,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:24,328 INFO [Thread-2452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2452. 2024-12-10T20:18:24,328 DEBUG [Thread-2452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:24,328 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:24,328 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:24,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:26,042 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-12-10T20:18:26,042 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:26,042 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:26,042 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:26,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:26,118 INFO [Thread-2450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2450. 2024-12-10T20:18:26,118 DEBUG [Thread-2450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T20:18:26,118 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:26,118 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:26,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-10T20:18:29,257 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2882 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2976 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3038 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2892 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2976 2024-12-10T20:18:29,257 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T20:18:29,257 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:18:29,257 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:18:29,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:29,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:29,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:29,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T20:18:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T20:18:29,261 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861909261"}]},"ts":"1733861909261"} 2024-12-10T20:18:29,263 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T20:18:29,263 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T20:18:29,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T20:18:29,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, UNASSIGN}] 2024-12-10T20:18:29,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, UNASSIGN 2024-12-10T20:18:29,267 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=148 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=CLOSING, regionLocation=03a2ae042c95,38133,1733861689839 2024-12-10T20:18:29,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, UNASSIGN because future has completed 2024-12-10T20:18:29,269 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:18:29,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839}] 2024-12-10T20:18:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T20:18:29,423 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(122): Close 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:29,423 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:18:29,423 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1722): Closing 4247c586a6941b4d0687e2fa49098b0a, disabling compactions & flushes 2024-12-10T20:18:29,423 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:29,423 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:29,423 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. after waiting 0 ms 2024-12-10T20:18:29,423 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:29,424 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(2902): Flushing 4247c586a6941b4d0687e2fa49098b0a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T20:18:29,424 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=A 2024-12-10T20:18:29,424 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:29,424 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=B 2024-12-10T20:18:29,424 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:29,424 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4247c586a6941b4d0687e2fa49098b0a, store=C 2024-12-10T20:18:29,424 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T20:18:29,431 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121023003644a41b4644b916133d35974b7c_4247c586a6941b4d0687e2fa49098b0a is 50, key is test_row_0/A:col10/1733861904287/Put/seqid=0 2024-12-10T20:18:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742397_1573 (size=12154) 2024-12-10T20:18:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T20:18:29,835 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:18:29,838 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121023003644a41b4644b916133d35974b7c_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121023003644a41b4644b916133d35974b7c_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:29,839 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/19884882af71415fae1e52fda7c69045, store: [table=TestAcidGuarantees family=A region=4247c586a6941b4d0687e2fa49098b0a] 2024-12-10T20:18:29,839 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/19884882af71415fae1e52fda7c69045 is 175, key is test_row_0/A:col10/1733861904287/Put/seqid=0 2024-12-10T20:18:29,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742398_1574 (size=30955) 2024-12-10T20:18:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T20:18:30,243 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/19884882af71415fae1e52fda7c69045 2024-12-10T20:18:30,252 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/650125089af3409c8e90c4c1dece1a8f is 50, key is test_row_0/B:col10/1733861904287/Put/seqid=0 2024-12-10T20:18:30,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742399_1575 (size=12001) 2024-12-10T20:18:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T20:18:30,659 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/650125089af3409c8e90c4c1dece1a8f 2024-12-10T20:18:30,665 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/80519df288084d86b295ad626ecc7a41 is 50, key is test_row_0/C:col10/1733861904287/Put/seqid=0 2024-12-10T20:18:30,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742400_1576 (size=12001) 2024-12-10T20:18:31,070 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/80519df288084d86b295ad626ecc7a41 2024-12-10T20:18:31,073 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/A/19884882af71415fae1e52fda7c69045 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/19884882af71415fae1e52fda7c69045 2024-12-10T20:18:31,075 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/19884882af71415fae1e52fda7c69045, entries=150, sequenceid=117, filesize=30.2 K 2024-12-10T20:18:31,076 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/B/650125089af3409c8e90c4c1dece1a8f as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/650125089af3409c8e90c4c1dece1a8f 2024-12-10T20:18:31,079 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/650125089af3409c8e90c4c1dece1a8f, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T20:18:31,079 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/.tmp/C/80519df288084d86b295ad626ecc7a41 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/80519df288084d86b295ad626ecc7a41 2024-12-10T20:18:31,082 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/80519df288084d86b295ad626ecc7a41, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T20:18:31,083 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4247c586a6941b4d0687e2fa49098b0a in 1660ms, sequenceid=117, compaction requested=true 2024-12-10T20:18:31,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436] to archive 2024-12-10T20:18:31,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:18:31,090 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/4baf7f9a01034525945e4ae675fa8436 2024-12-10T20:18:31,090 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/d95eab1aa583474c8f57e4dd537a4988 2024-12-10T20:18:31,090 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/07875ea8c1854419a6bb8f3f7eebb4d5 2024-12-10T20:18:31,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/0c1da7c09fc54324b187954beabe365d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/c181830171194eeabec2bace3a170157, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/19bcc28170764c7b8c430249e3151dc2] to archive 2024-12-10T20:18:31,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:18:31,093 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/c181830171194eeabec2bace3a170157 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/c181830171194eeabec2bace3a170157 2024-12-10T20:18:31,093 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/0c1da7c09fc54324b187954beabe365d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/0c1da7c09fc54324b187954beabe365d 2024-12-10T20:18:31,093 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/19bcc28170764c7b8c430249e3151dc2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/19bcc28170764c7b8c430249e3151dc2 2024-12-10T20:18:31,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/56abf6cdf2834e85b55d705d7d5367b2, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/f602202bac87428fa409eae9ca7c156d, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/87053aca3e054f5bac33c092790c3198] to archive 2024-12-10T20:18:31,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:18:31,097 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/87053aca3e054f5bac33c092790c3198 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/87053aca3e054f5bac33c092790c3198 2024-12-10T20:18:31,097 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/56abf6cdf2834e85b55d705d7d5367b2 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/56abf6cdf2834e85b55d705d7d5367b2 2024-12-10T20:18:31,099 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/f602202bac87428fa409eae9ca7c156d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/f602202bac87428fa409eae9ca7c156d 2024-12-10T20:18:31,103 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/recovered.edits/120.seqid, newMaxSeqId=120, maxSeqId=4 2024-12-10T20:18:31,103 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. 2024-12-10T20:18:31,103 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1676): Region close journal for 4247c586a6941b4d0687e2fa49098b0a: Waiting for close lock at 1733861909423Running coprocessor pre-close hooks at 1733861909423Disabling compacts and flushes for region at 1733861909423Disabling writes for close at 1733861909423Obtaining lock to block concurrent updates at 1733861909424 (+1 ms)Preparing flush snapshotting stores in 4247c586a6941b4d0687e2fa49098b0a at 1733861909424Finished memstore snapshotting TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733861909424Flushing stores of TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a. at 1733861909425 (+1 ms)Flushing 4247c586a6941b4d0687e2fa49098b0a/A: creating writer at 1733861909425Flushing mob file 4247c586a6941b4d0687e2fa49098b0a/A: appending metadata at 1733861909431 (+6 ms)Flushing mob file 4247c586a6941b4d0687e2fa49098b0a/A: closing flushed file at 1733861909431Flushing 4247c586a6941b4d0687e2fa49098b0a/A: appending metadata at 1733861909839 (+408 ms)Flushing 4247c586a6941b4d0687e2fa49098b0a/A: closing flushed file at 1733861909839Flushing 4247c586a6941b4d0687e2fa49098b0a/B: creating writer at 1733861910248 (+409 ms)Flushing 4247c586a6941b4d0687e2fa49098b0a/B: appending metadata at 1733861910251 (+3 ms)Flushing 4247c586a6941b4d0687e2fa49098b0a/B: closing flushed file at 1733861910251Flushing 4247c586a6941b4d0687e2fa49098b0a/C: creating writer at 1733861910662 (+411 ms)Flushing 4247c586a6941b4d0687e2fa49098b0a/C: appending metadata at 1733861910665 (+3 ms)Flushing 4247c586a6941b4d0687e2fa49098b0a/C: closing flushed file at 1733861910665Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fbaa59f: reopening flushed file at 1733861911072 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b8de379: reopening flushed file at 1733861911075 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e7a9bc0: reopening flushed file at 1733861911079 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4247c586a6941b4d0687e2fa49098b0a in 1660ms, sequenceid=117, compaction requested=true at 1733861911083 (+4 ms)Writing region close event to WAL at 1733861911100 (+17 ms)Running coprocessor post-close hooks at 1733861911103 (+3 ms)Closed at 1733861911103 2024-12-10T20:18:31,105 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(157): Closed 4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:31,106 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=148 updating hbase:meta row=4247c586a6941b4d0687e2fa49098b0a, regionState=CLOSED 2024-12-10T20:18:31,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 because future has completed 2024-12-10T20:18:31,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-10T20:18:31,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; CloseRegionProcedure 4247c586a6941b4d0687e2fa49098b0a, server=03a2ae042c95,38133,1733861689839 in 1.8380 sec 2024-12-10T20:18:31,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-10T20:18:31,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4247c586a6941b4d0687e2fa49098b0a, UNASSIGN in 1.8440 sec 2024-12-10T20:18:31,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-10T20:18:31,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8460 sec 2024-12-10T20:18:31,113 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733861911112"}]},"ts":"1733861911112"} 2024-12-10T20:18:31,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T20:18:31,114 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T20:18:31,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-10T20:18:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T20:18:31,418 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:18:31,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T20:18:34,842 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_417660294_22 at /127.0.0.1:51444 [Receiving block BP-1506670894-172.17.0.3-1733861683702:blk_1073742392_1568] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 3421ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2/, blockId=1073742392, seqno=42 2024-12-10T20:18:36,622 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:18:39,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T20:18:40,483 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_417660294_22 at /127.0.0.1:51444 [Receiving block BP-1506670894-172.17.0.3-1733861683702:blk_1073742392_1568] {}] datanode.BlockReceiver(464): Slow flushOrSync took 5641ms (threshold=300ms), isSync:false, flushTotalNanos=5640154392ns, volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2/, blockId=1073742392, seqno=42 2024-12-10T20:18:40,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK], data length=767, duration=9064ms, unfinishedReplicas=0, lastAckTimestamp=-1, monitor name: defaultMonitorName 2024-12-10T20:18:40,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] monitor.ExcludeDatanodeManager(82): Added datanode: DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK] to exclude cache by [slow packet ack] success, current excludeDNsCache size=1 2024-12-10T20:18:40,485 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1368): Slow sync cost: 9064 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:18:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:40,488 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=150, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T20:18:40,489 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=150, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:40,492 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,494 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C, FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/recovered.edits] 2024-12-10T20:18:40,497 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/3b531e5fae514613b34df90bacca1b11 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/3b531e5fae514613b34df90bacca1b11 2024-12-10T20:18:40,497 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/97cc9330c68046d495bfe1a5283dbbad to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/97cc9330c68046d495bfe1a5283dbbad 2024-12-10T20:18:40,497 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/a028360126b74ade9d2e749e04434ee9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/a028360126b74ade9d2e749e04434ee9 2024-12-10T20:18:40,497 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/19884882af71415fae1e52fda7c69045 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/A/19884882af71415fae1e52fda7c69045 2024-12-10T20:18:40,502 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/69f82218bfd64ea1b0792971d72435a9 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/69f82218bfd64ea1b0792971d72435a9 2024-12-10T20:18:40,502 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/646fcb6ea731499db54db0ecda589471 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/646fcb6ea731499db54db0ecda589471 2024-12-10T20:18:40,502 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/650125089af3409c8e90c4c1dece1a8f to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/650125089af3409c8e90c4c1dece1a8f 2024-12-10T20:18:40,502 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/e7eecb9efb8c42daab5c1daf5f937f48 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/B/e7eecb9efb8c42daab5c1daf5f937f48 2024-12-10T20:18:40,505 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/15668c072ead4138b159f2c1feef25b5 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/15668c072ead4138b159f2c1feef25b5 2024-12-10T20:18:40,505 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/80519df288084d86b295ad626ecc7a41 to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/80519df288084d86b295ad626ecc7a41 2024-12-10T20:18:40,505 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/d9ce93fdd0ad43dda1293082b376e58d to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/d9ce93fdd0ad43dda1293082b376e58d 2024-12-10T20:18:40,505 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/e43e5a9ecda642479707e83d5f6dd26c to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/C/e43e5a9ecda642479707e83d5f6dd26c 2024-12-10T20:18:40,507 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/recovered.edits/120.seqid to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a/recovered.edits/120.seqid 2024-12-10T20:18:40,507 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/default/TestAcidGuarantees/4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,507 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T20:18:40,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T20:18:40,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T20:18:40,511 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100dd0f6904f144e928e75193ea2bef751_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100dd0f6904f144e928e75193ea2bef751_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,511 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121023003644a41b4644b916133d35974b7c_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121023003644a41b4644b916133d35974b7c_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,511 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121025eb186aa1804f7da0be08059f95f49e_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121025eb186aa1804f7da0be08059f95f49e_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,511 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d151bcdf70974628936edab91fb3ed21_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210d151bcdf70974628936edab91fb3ed21_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,511 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fb5dacbf418641eb986d62bc5a22b8de_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fb5dacbf418641eb986d62bc5a22b8de_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,511 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121041fde26bc9244777aa4048e266f16f0e_4247c586a6941b4d0687e2fa49098b0a to hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121041fde26bc9244777aa4048e266f16f0e_4247c586a6941b4d0687e2fa49098b0a 2024-12-10T20:18:40,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T20:18:40,513 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=150, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:40,515 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T20:18:40,516 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T20:18:40,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=150, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:40,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T20:18:40,518 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733861920517"}]},"ts":"9223372036854775807"} 2024-12-10T20:18:40,519 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T20:18:40,519 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4247c586a6941b4d0687e2fa49098b0a, NAME => 'TestAcidGuarantees,,1733861862723.4247c586a6941b4d0687e2fa49098b0a.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T20:18:40,519 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T20:18:40,519 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733861920519"}]},"ts":"9223372036854775807"} 2024-12-10T20:18:40,521 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T20:18:40,521 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=150, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T20:18:40,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 9.1020 sec 2024-12-10T20:18:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-10T20:18:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T20:18:40,598 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T20:18:40,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=253 (was 251) - Thread LEAK? -, OpenFileDescriptor=439 (was 441), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=878 (was 859) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=705 (was 1272) 2024-12-10T20:18:40,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T20:18:40,610 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:18:40,611 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:18:40,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:40,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:40,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:18:40,611 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T20:18:40,611 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1945866401, stopped=false 2024-12-10T20:18:40,612 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=03a2ae042c95,45643,1733861688870 2024-12-10T20:18:41,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T20:18:41,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T20:18:41,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:18:41,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:18:41,283 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T20:18:41,285 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:18:41,285 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:18:41,285 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:18:41,285 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:18:41,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:41,287 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '03a2ae042c95,38133,1733861689839' ***** 2024-12-10T20:18:41,287 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T20:18:41,287 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T20:18:41,288 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T20:18:41,288 INFO [RS:0;03a2ae042c95:38133 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(959): stopping server 03a2ae042c95,38133,1733861689839 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;03a2ae042c95:38133. 2024-12-10T20:18:41,289 DEBUG [RS:0;03a2ae042c95:38133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:18:41,289 DEBUG [RS:0;03a2ae042c95:38133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T20:18:41,289 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T20:18:41,290 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T20:18:41,290 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T20:18:41,290 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T20:18:41,291 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:41,291 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T20:18:41,291 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T20:18:41,291 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T20:18:41,291 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T20:18:41,291 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T20:18:41,291 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-10T20:18:41,298 INFO [regionserver/03a2ae042c95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:18:41,312 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/info/fc7eebdccd6c46e1b6a6f6db63a20965 is 95, key is TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4./info:/1733861727739/DeleteFamily/seqid=0 2024-12-10T20:18:41,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742401_1577 (size=6514) 2024-12-10T20:18:41,491 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:41,691 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:41,717 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/info/fc7eebdccd6c46e1b6a6f6db63a20965 2024-12-10T20:18:41,741 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/ns/2a16241ab70740a4962fc044974c0c73 is 93, key is TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4./ns:/1733861727704/DeleteFamily/seqid=0 2024-12-10T20:18:41,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742402_1578 (size=5906) 2024-12-10T20:18:41,892 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:41,966 INFO [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T20:18:41,967 INFO [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T20:18:42,092 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:42,145 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/ns/2a16241ab70740a4962fc044974c0c73 2024-12-10T20:18:42,172 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/rep_barrier/bc64ec6aef594025a058fd0ebf19f358 is 102, key is TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4./rep_barrier:/1733861727704/DeleteFamily/seqid=0 2024-12-10T20:18:42,292 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:42,493 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:42,693 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:42,893 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:43,094 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:43,294 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:43,494 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:43,695 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:43,895 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:44,095 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:44,296 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:44,496 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:44,696 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:44,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:18:44,897 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:45,097 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:45,297 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:45,497 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:45,698 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:45,898 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:46,098 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:46,299 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:46,499 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:46,699 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:46,900 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:47,100 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:47,300 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:47,500 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:47,701 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:47,801 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:18:47,901 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:48,101 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:48,302 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:48,502 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:48,702 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:48,903 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:49,103 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:49,303 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:49,504 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:49,704 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:49,904 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:50,104 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:50,305 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:50,505 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:50,706 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:50,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742403_1579 (size=6025) 2024-12-10T20:18:50,906 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:51,106 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:51,121 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/rep_barrier/bc64ec6aef594025a058fd0ebf19f358 2024-12-10T20:18:51,148 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/table/a0296c106fa940a9b995ec9649c6a16e is 96, key is TestAcidGuarantees,,1733861693355.308bff2a777bdccf66ea752f9872cdd4./table:/1733861727704/DeleteFamily/seqid=0 2024-12-10T20:18:51,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742404_1580 (size=5936) 2024-12-10T20:18:51,306 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:51,507 DEBUG [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:18:51,554 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/table/a0296c106fa940a9b995ec9649c6a16e 2024-12-10T20:18:51,563 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/info/fc7eebdccd6c46e1b6a6f6db63a20965 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/info/fc7eebdccd6c46e1b6a6f6db63a20965 2024-12-10T20:18:51,567 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/info/fc7eebdccd6c46e1b6a6f6db63a20965, entries=12, sequenceid=90, filesize=6.4 K 2024-12-10T20:18:51,568 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/ns/2a16241ab70740a4962fc044974c0c73 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/ns/2a16241ab70740a4962fc044974c0c73 2024-12-10T20:18:51,570 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/ns/2a16241ab70740a4962fc044974c0c73, entries=8, sequenceid=90, filesize=5.8 K 2024-12-10T20:18:51,571 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/rep_barrier/bc64ec6aef594025a058fd0ebf19f358 as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/rep_barrier/bc64ec6aef594025a058fd0ebf19f358 2024-12-10T20:18:51,573 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/rep_barrier/bc64ec6aef594025a058fd0ebf19f358, entries=6, sequenceid=90, filesize=5.9 K 2024-12-10T20:18:51,574 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/.tmp/table/a0296c106fa940a9b995ec9649c6a16e as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/table/a0296c106fa940a9b995ec9649c6a16e 2024-12-10T20:18:51,576 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/table/a0296c106fa940a9b995ec9649c6a16e, entries=7, sequenceid=90, filesize=5.8 K 2024-12-10T20:18:51,577 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 10286ms, sequenceid=90, compaction requested=false 2024-12-10T20:18:51,580 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-10T20:18:51,580 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T20:18:51,581 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T20:18:51,581 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733861921290Running coprocessor pre-close hooks at 1733861921291 (+1 ms)Disabling compacts and flushes for region at 1733861921291Disabling writes for close at 1733861921291Obtaining lock to block concurrent updates at 1733861921291Preparing flush snapshotting stores in 1588230740 at 1733861921291Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733861921292 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733861921293 (+1 ms)Flushing 1588230740/info: creating writer at 1733861921293Flushing 1588230740/info: appending metadata at 1733861921312 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733861921312Flushing 1588230740/ns: creating writer at 1733861921725 (+413 ms)Flushing 1588230740/ns: appending metadata at 1733861921741 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733861921741Flushing 1588230740/rep_barrier: creating writer at 1733861922155 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733861922172 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733861922172Flushing 1588230740/table: creating writer at 1733861931129 (+8957 ms)Flushing 1588230740/table: appending metadata at 1733861931148 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733861931148Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e62a9be: reopening flushed file at 1733861931562 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f88e1d0: reopening flushed file at 1733861931567 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@595a5b11: reopening flushed file at 1733861931570 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65a8dbe7: reopening flushed file at 1733861931573 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 10286ms, sequenceid=90, compaction requested=false at 1733861931577 (+4 ms)Writing region close event to WAL at 1733861931578 (+1 ms)Running coprocessor post-close hooks at 1733861931580 (+2 ms)Closed at 1733861931581 (+1 ms) 2024-12-10T20:18:51,581 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T20:18:51,707 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(976): stopping server 03a2ae042c95,38133,1733861689839; all regions closed. 2024-12-10T20:18:51,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741834_1010 (size=25344) 2024-12-10T20:18:51,716 DEBUG [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/oldWALs 2024-12-10T20:18:51,716 INFO [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 03a2ae042c95%2C38133%2C1733861689839.meta:.meta(num 1733861692784) 2024-12-10T20:18:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073741833_1009 (size=14982906) 2024-12-10T20:18:51,721 DEBUG [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/oldWALs 2024-12-10T20:18:51,721 INFO [RS:0;03a2ae042c95:38133 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 03a2ae042c95%2C38133%2C1733861689839:(num 1733861692127) 2024-12-10T20:18:51,721 DEBUG [RS:0;03a2ae042c95:38133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:18:51,721 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:18:51,721 INFO [RS:0;03a2ae042c95:38133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T20:18:51,721 INFO [RS:0;03a2ae042c95:38133 {}] hbase.ChoreService(370): Chore service for: regionserver/03a2ae042c95:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T20:18:51,721 INFO [RS:0;03a2ae042c95:38133 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T20:18:51,721 INFO [regionserver/03a2ae042c95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T20:18:51,722 INFO [RS:0;03a2ae042c95:38133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38133 2024-12-10T20:18:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T20:18:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/03a2ae042c95,38133,1733861689839 2024-12-10T20:18:54,162 INFO [RS:0;03a2ae042c95:38133 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T20:18:54,162 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [03a2ae042c95,38133,1733861689839] 2024-12-10T20:18:57,097 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:18:59,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T20:18:59,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T20:19:03,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:19:03,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38133-0x100115dc9260001, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:19:03,755 INFO [RS:0;03a2ae042c95:38133 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T20:19:03,755 INFO [RS:0;03a2ae042c95:38133 {}] regionserver.HRegionServer(1031): Exiting; stopping=03a2ae042c95,38133,1733861689839; zookeeper connection closed. 2024-12-10T20:19:03,756 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a630727 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a630727 2024-12-10T20:19:03,757 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-10T20:19:03,825 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/03a2ae042c95,38133,1733861689839 already deleted, retry=false 2024-12-10T20:19:03,825 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 03a2ae042c95,38133,1733861689839 expired; onlineServers=0 2024-12-10T20:19:03,826 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '03a2ae042c95,45643,1733861688870' ***** 2024-12-10T20:19:03,826 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T20:19:03,826 INFO [M:0;03a2ae042c95:45643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T20:19:03,827 INFO [M:0;03a2ae042c95:45643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T20:19:03,827 DEBUG [M:0;03a2ae042c95:45643 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T20:19:03,827 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T20:19:03,828 DEBUG [M:0;03a2ae042c95:45643 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T20:19:03,828 DEBUG [master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.large.0-1733861691801 {}] cleaner.HFileCleaner(306): Exit Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.large.0-1733861691801,5,FailOnTimeoutGroup] 2024-12-10T20:19:03,828 DEBUG [master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.small.0-1733861691801 {}] cleaner.HFileCleaner(306): Exit Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.small.0-1733861691801,5,FailOnTimeoutGroup] 2024-12-10T20:19:03,828 INFO [M:0;03a2ae042c95:45643 {}] hbase.ChoreService(370): Chore service for: master/03a2ae042c95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T20:19:03,828 INFO [M:0;03a2ae042c95:45643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T20:19:03,829 DEBUG [M:0;03a2ae042c95:45643 {}] master.HMaster(1795): Stopping service threads 2024-12-10T20:19:03,829 INFO [M:0;03a2ae042c95:45643 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T20:19:03,829 INFO [M:0;03a2ae042c95:45643 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T20:19:03,830 INFO [M:0;03a2ae042c95:45643 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T20:19:03,830 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T20:19:03,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T20:19:03,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:19:03,945 DEBUG [M:0;03a2ae042c95:45643 {}] zookeeper.ZKUtil(347): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T20:19:03,945 WARN [M:0;03a2ae042c95:45643 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T20:19:03,948 INFO [M:0;03a2ae042c95:45643 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/.lastflushedseqids 2024-12-10T20:19:03,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742405_1581 (size=72) 2024-12-10T20:19:04,363 INFO [M:0;03a2ae042c95:45643 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T20:19:04,363 INFO [M:0;03a2ae042c95:45643 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T20:19:04,364 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T20:19:04,364 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:19:04,364 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:19:04,364 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T20:19:04,364 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:19:04,364 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=726.12 KB heapSize=886.16 KB 2024-12-10T20:19:04,568 INFO [AsyncFSWAL-0-hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData-prefix:03a2ae042c95,45643,1733861688870 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46079,DS-68733e57-d57c-4e39-97e9-323c09730199,DISK]] 2024-12-10T20:19:04,588 DEBUG [M:0;03a2ae042c95:45643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06be234087ab495aa9b87caabcd9c6af is 82, key is hbase:meta,,1/info:regioninfo/1733861692884/Put/seqid=0 2024-12-10T20:19:04,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742406_1582 (size=5672) 2024-12-10T20:19:04,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:19:04,992 INFO [M:0;03a2ae042c95:45643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1976 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06be234087ab495aa9b87caabcd9c6af 2024-12-10T20:19:05,015 DEBUG [M:0;03a2ae042c95:45643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ab8293b629a4ec88f6ba27453f0c1fe is 2480, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1733861730963/Put/seqid=0 2024-12-10T20:19:05,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742407_1583 (size=41928) 2024-12-10T20:19:05,419 INFO [M:0;03a2ae042c95:45643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=725.56 KB at sequenceid=1976 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ab8293b629a4ec88f6ba27453f0c1fe 2024-12-10T20:19:05,422 INFO [M:0;03a2ae042c95:45643 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ab8293b629a4ec88f6ba27453f0c1fe 2024-12-10T20:19:05,439 DEBUG [M:0;03a2ae042c95:45643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56ab34a33cb841239f2c826c3e5f425d is 69, key is 03a2ae042c95,38133,1733861689839/rs:state/1733861691869/Put/seqid=0 2024-12-10T20:19:05,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742408_1584 (size=5156) 2024-12-10T20:19:05,843 INFO [M:0;03a2ae042c95:45643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1976 (bloomFilter=true), to=hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56ab34a33cb841239f2c826c3e5f425d 2024-12-10T20:19:05,850 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06be234087ab495aa9b87caabcd9c6af as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/06be234087ab495aa9b87caabcd9c6af 2024-12-10T20:19:05,855 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/06be234087ab495aa9b87caabcd9c6af, entries=8, sequenceid=1976, filesize=5.5 K 2024-12-10T20:19:05,856 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ab8293b629a4ec88f6ba27453f0c1fe as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ab8293b629a4ec88f6ba27453f0c1fe 2024-12-10T20:19:05,860 INFO [M:0;03a2ae042c95:45643 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ab8293b629a4ec88f6ba27453f0c1fe 2024-12-10T20:19:05,860 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ab8293b629a4ec88f6ba27453f0c1fe, entries=150, sequenceid=1976, filesize=40.9 K 2024-12-10T20:19:05,861 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56ab34a33cb841239f2c826c3e5f425d as hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/56ab34a33cb841239f2c826c3e5f425d 2024-12-10T20:19:05,863 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35331/user/jenkins/test-data/3911a73e-8a28-6658-1351-e0c7681d2d2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/56ab34a33cb841239f2c826c3e5f425d, entries=1, sequenceid=1976, filesize=5.0 K 2024-12-10T20:19:05,864 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(3140): Finished flush of dataSize ~726.12 KB/743543, heapSize ~885.86 KB/907120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1500ms, sequenceid=1976, compaction requested=false 2024-12-10T20:19:05,865 INFO [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:19:05,865 DEBUG [M:0;03a2ae042c95:45643 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733861944364Disabling compacts and flushes for region at 1733861944364Disabling writes for close at 1733861944364Obtaining lock to block concurrent updates at 1733861944364Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733861944364Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=743543, getHeapSize=907360, getOffHeapSize=0, getCellsCount=2151 at 1733861944365 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733861944568 (+203 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733861944569 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733861944587 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733861944587Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733861944998 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733861945015 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733861945015Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733861945422 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733861945438 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733861945438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77da5173: reopening flushed file at 1733861945849 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c9edb7e: reopening flushed file at 1733861945856 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fee8794: reopening flushed file at 1733861945860 (+4 ms)Finished flush of dataSize ~726.12 KB/743543, heapSize ~885.86 KB/907120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1500ms, sequenceid=1976, compaction requested=false at 1733861945864 (+4 ms)Writing region close event to WAL at 1733861945865 (+1 ms)Closed at 1733861945865 2024-12-10T20:19:05,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46079 is added to blk_1073742392_1568 (size=21632) 2024-12-10T20:19:05,867 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T20:19:05,867 INFO [M:0;03a2ae042c95:45643 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T20:19:05,867 INFO [M:0;03a2ae042c95:45643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45643 2024-12-10T20:19:05,867 INFO [M:0;03a2ae042c95:45643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T20:19:06,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:19:06,195 INFO [M:0;03a2ae042c95:45643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T20:19:06,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45643-0x100115dc9260000, quorum=127.0.0.1:62580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:19:06,293 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:19:06,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T20:19:06,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T20:19:06,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T20:19:06,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/hadoop.log.dir/,STOPPED} 2024-12-10T20:19:06,301 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T20:19:06,301 WARN [BP-1506670894-172.17.0.3-1733861683702 heartbeating to localhost/127.0.0.1:35331 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T20:19:06,301 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T20:19:06,301 WARN [BP-1506670894-172.17.0.3-1733861683702 heartbeating to localhost/127.0.0.1:35331 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1506670894-172.17.0.3-1733861683702 (Datanode Uuid cd217020-ad99-425b-ab5c-371e4fd24d2e) service to localhost/127.0.0.1:35331 2024-12-10T20:19:06,303 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data1/current/BP-1506670894-172.17.0.3-1733861683702 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:19:06,303 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/cluster_db429a9a-4959-cf90-d2c8-9449a60ac8f7/data/data2/current/BP-1506670894-172.17.0.3-1733861683702 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:19:06,303 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T20:19:06,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T20:19:06,309 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T20:19:06,309 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T20:19:06,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T20:19:06,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/28768769-451a-132d-2ea1-a1d04fad6fb6/hadoop.log.dir/,STOPPED} 2024-12-10T20:19:06,323 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T20:19:06,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down